NGramsCounts

A simple transformer that represents each ngram as an NGram and counts their occurrence. Returns an RDD[(NGram, Int)] that is sorted by frequency in descending order.

This implementation may not be space-efficient, but should handle commonly-sized workloads well.

mode: a control flag defined in NGramsCountsMode

Linear Supertypes

Product, Equals, FunctionNode[RDD[Seq[Seq[T]]], RDD[(NGram[T], Int)]], Serializable, Serializable, (RDD[Seq[Seq[T]]]) ⇒ RDD[(NGram[T], Int)], AnyRef, Any

Instance Constructors

new NGramsCounts(mode: NGramsCountsMode.Value = NGramsCountsMode.Default)(implicit arg0: ClassTag[T])

mode
a control flag defined in NGramsCountsMode

Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
def andThen[A](g: (RDD[(NGram[T], Int)]) ⇒ A): (RDD[Seq[Seq[T]]]) ⇒ A

Definition Classes
Function1
Annotations
@unspecialized()
def apply(rdd: RDD[Seq[Seq[T]]]): RDD[(NGram[T], Int)]

Definition Classes
NGramsCounts → Function1
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def compose[A](g: (A) ⇒ RDD[Seq[Seq[T]]]): (A) ⇒ RDD[(NGram[T], Int)]

Definition Classes
Function1
Annotations
@unspecialized()
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
val mode: NGramsCountsMode.Value

a control flag defined in NGramsCountsMode
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
Function1 → AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

case class NGramsCounts[T](mode: NGramsCountsMode.Value = NGramsCountsMode.Default)(implicit evidence$3: ClassTag[T]) extends FunctionNode[RDD[Seq[Seq[T]]], RDD[(NGram[T], Int)]] with Product with Serializable

Instance Constructors

new NGramsCounts(mode: NGramsCountsMode.Value = NGramsCountsMode.Default)(implicit arg0: ClassTag[T])

Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

def andThen[A](g: (RDD[(NGram[T], Int)]) ⇒ A): (RDD[Seq[Seq[T]]]) ⇒ A

def apply(rdd: RDD[Seq[Seq[T]]]): RDD[(NGram[T], Int)]

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def compose[A](g: (A) ⇒ RDD[Seq[Seq[T]]]): (A) ⇒ RDD[(NGram[T], Int)]

final def eq(arg0: AnyRef): Boolean

def finalize(): Unit

final def getClass(): Class[_]

final def isInstanceOf[T0]: Boolean

val mode: NGramsCountsMode.Value

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Product

Inherited from Equals

Inherited from FunctionNode[RDD[Seq[Seq[T]]], RDD[(NGram[T], Int)]]

Inherited from Serializable

Inherited from Serializable

Inherited from (RDD[Seq[Seq[T]]]) ⇒ RDD[(NGram[T], Int)]

Inherited from AnyRef

Inherited from Any

Ungrouped