Shuffler

Instance Constructors

new Shuffler(numParts: Option[Int] = scala.None)(implicit arg0: ClassTag[T])

numParts
An optional parameter indicating the number of output partitions.

Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
final def andThen[C, L](est: LabelEstimator[T, C, L], data: PipelineDataset[T], labels: PipelineDataset[L]): Pipeline[T, C]

Chains a label estimator onto the end of this pipeline, producing a new pipeline.
Chains a label estimator onto the end of this pipeline, producing a new pipeline. If this pipeline has already been executed, it will not need to be fit again.
est
The estimator to chain onto the end of this pipeline
data
The training data to use (the estimator will be fit on the result of passing this data through the current pipeline)
labels
The labels to use when fitting the LabelEstimator. Must be zippable with the training data.

Definition Classes
Chainable
final def andThen[C, L](est: LabelEstimator[T, C, L], data: RDD[T], labels: PipelineDataset[L]): Pipeline[T, C]

Chains a label estimator onto the end of this pipeline, producing a new pipeline.
Chains a label estimator onto the end of this pipeline, producing a new pipeline. If this pipeline has already been executed, it will not need to be fit again.
est
The estimator to chain onto the end of this pipeline
data
The training data to use (the estimator will be fit on the result of passing this data through the current pipeline)
labels
The labels to use when fitting the LabelEstimator. Must be zippable with the training data.

Definition Classes
Chainable
final def andThen[C, L](est: LabelEstimator[T, C, L], data: PipelineDataset[T], labels: RDD[L]): Pipeline[T, C]

Chains a label estimator onto the end of this pipeline, producing a new pipeline.
Chains a label estimator onto the end of this pipeline, producing a new pipeline. If this pipeline has already been executed, it will not need to be fit again.
est
The estimator to chain onto the end of this pipeline
data
The training data to use (the estimator will be fit on the result of passing this data through the current pipeline)
labels
The labels to use when fitting the LabelEstimator. Must be zippable with the training data.

Definition Classes
Chainable
final def andThen[C, L](est: LabelEstimator[T, C, L], data: RDD[T], labels: RDD[L]): Pipeline[T, C]

Chains a label estimator onto the end of this pipeline, producing a new pipeline.
Chains a label estimator onto the end of this pipeline, producing a new pipeline. If this pipeline has already been executed, it will not need to be fit again.
est
The estimator to chain onto the end of this pipeline
data
The training data to use (the estimator will be fit on the result of passing this data through the current pipeline)
labels
The labels to use when fitting the LabelEstimator. Must be zippable with the training data.

Definition Classes
Chainable
final def andThen[C](est: Estimator[T, C], data: PipelineDataset[T]): Pipeline[T, C]

Chains an estimator onto the end of this pipeline, producing a new pipeline.
Chains an estimator onto the end of this pipeline, producing a new pipeline. If this pipeline has already been executed, it will not need to be fit again.
est
The estimator to chain onto the end of this pipeline
data
The training data to use (the estimator will be fit on the result of passing this data through the current pipeline)

Definition Classes
Chainable
final def andThen[C](est: Estimator[T, C], data: RDD[T]): Pipeline[T, C]

Chains an estimator onto the end of this pipeline, producing a new pipeline.
Chains an estimator onto the end of this pipeline, producing a new pipeline. If this pipeline has already been executed, it will not need to be fit again.
est
The estimator to chain onto the end of this pipeline
data
The training data to use (the estimator will be fit on the result of passing this data through the current pipeline)

Definition Classes
Chainable
final def andThen[C](next: Chainable[T, C]): Pipeline[T, C]

Chains a pipeline onto the end of this one, producing a new pipeline.
Chains a pipeline onto the end of this one, producing a new pipeline. If either this pipeline or the following has already been executed, it will not need to be fit again.
next
the pipeline to chain

Definition Classes
Chainable
def apply(in: T): T

The application of this Transformer to a single input item.
The application of this Transformer to a single input item. This method MUST be overridden by ML developers.
in
The input item to pass into this transformer
returns
The output value

Definition Classes
Shuffler → Transformer
def apply(in: RDD[T]): RDD[T]

The application of this Transformer to an RDD of input items.
The application of this Transformer to an RDD of input items. This method may optionally be overridden by ML developers.
in
The bulk RDD input to pass into this transformer
returns
The bulk RDD output for the given input

Definition Classes
Shuffler → Transformer
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def execute(deps: Seq[Expression]): Expression

Definition Classes
TransformerOperator → Operator
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def label: String

Definition Classes
Operator
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toPipeline: Pipeline[T, T]

A method that converts this object into a Pipeline.
A method that converts this object into a Pipeline. Must be implemented by anything that extends Chainable.

Definition Classes
Transformer → Chainable
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

class Shuffler[T] extends Transformer[T, T] with Logging

Instance Constructors

new Shuffler(numParts: Option[Int] = scala.None)(implicit arg0: ClassTag[T])

Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

final def andThen[C, L](est: LabelEstimator[T, C, L], data: PipelineDataset[T], labels: PipelineDataset[L]): Pipeline[T, C]

final def andThen[C, L](est: LabelEstimator[T, C, L], data: RDD[T], labels: PipelineDataset[L]): Pipeline[T, C]

final def andThen[C, L](est: LabelEstimator[T, C, L], data: PipelineDataset[T], labels: RDD[L]): Pipeline[T, C]

final def andThen[C, L](est: LabelEstimator[T, C, L], data: RDD[T], labels: RDD[L]): Pipeline[T, C]

final def andThen[C](est: Estimator[T, C], data: PipelineDataset[T]): Pipeline[T, C]

final def andThen[C](est: Estimator[T, C], data: RDD[T]): Pipeline[T, C]

final def andThen[C](next: Chainable[T, C]): Pipeline[T, C]

def apply(in: T): T

def apply(in: RDD[T]): RDD[T]

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def execute(deps: Seq[Expression]): Expression

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def label: String

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toPipeline: Pipeline[T, T]

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Logging

Inherited from Transformer[T, T]

Inherited from Chainable[T, T]

Inherited from TransformerOperator

Inherited from Serializable

Inherited from Serializable

Inherited from Operator

Inherited from AnyRef

Inherited from Any

Ungrouped