abstract class SQLImplicits extends EncoderImplicits with Serializable
A collection of implicit methods for converting common Scala objects into org.apache.spark.sql.Datasets.
- Source
- SQLImplicits.scala
- Since
1.6.0
- Alphabetic
- By Inheritance
- SQLImplicits
- EncoderImplicits
- Serializable
- LowPrioritySQLImplicits
- AnyRef
- Any
- Hide All
- Show All
- Public
- Protected
Instance Constructors
- new SQLImplicits()
Type Members
- implicit class StringToColumn extends AnyRef
Converts $"col name" into a org.apache.spark.sql.Column.
Converts $"col name" into a org.apache.spark.sql.Column.
- Since
2.0.0
Abstract Value Members
- implicit abstract def localSeqToDatasetHolder[T](s: Seq[T])(implicit arg0: Encoder[T]): DatasetHolder[T]
Creates a Dataset from a local Seq.
Creates a Dataset from a local Seq.
- Since
1.6.0
- implicit abstract def rddToDatasetHolder[T](rdd: RDD[T])(implicit arg0: Encoder[T]): DatasetHolder[T]
Creates a Dataset from an RDD.
Creates a Dataset from an RDD.
- Since
1.6.0
- abstract def session: SparkSession
- Attributes
- protected
Concrete Value Members
- final def !=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def ##: Int
- Definition Classes
- AnyRef → Any
- final def ==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def asInstanceOf[T0]: T0
- Definition Classes
- Any
- def clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.CloneNotSupportedException]) @IntrinsicCandidate() @native()
- final def eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- def equals(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef → Any
- final def getClass(): Class[_ <: AnyRef]
- Definition Classes
- AnyRef → Any
- Annotations
- @IntrinsicCandidate() @native()
- def hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @IntrinsicCandidate() @native()
- final def isInstanceOf[T0]: Boolean
- Definition Classes
- Any
- final def ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- implicit val newBooleanArrayEncoder: Encoder[Array[Boolean]]
- Definition Classes
- EncoderImplicits
- Since
1.6.1
- implicit def newBooleanEncoder: Encoder[Boolean]
- Definition Classes
- EncoderImplicits
- Since
1.6.0
- implicit def newBoxedBooleanEncoder: Encoder[Boolean]
- Definition Classes
- EncoderImplicits
- Since
2.0.0
- implicit def newBoxedByteEncoder: Encoder[Byte]
- Definition Classes
- EncoderImplicits
- Since
2.0.0
- implicit def newBoxedDoubleEncoder: Encoder[Double]
- Definition Classes
- EncoderImplicits
- Since
2.0.0
- implicit def newBoxedFloatEncoder: Encoder[Float]
- Definition Classes
- EncoderImplicits
- Since
2.0.0
- implicit def newBoxedIntEncoder: Encoder[Integer]
- Definition Classes
- EncoderImplicits
- Since
2.0.0
- implicit def newBoxedLongEncoder: Encoder[Long]
- Definition Classes
- EncoderImplicits
- Since
2.0.0
- implicit def newBoxedShortEncoder: Encoder[Short]
- Definition Classes
- EncoderImplicits
- Since
2.0.0
- implicit val newByteArrayEncoder: Encoder[Array[Byte]]
- Definition Classes
- EncoderImplicits
- Since
1.6.1
- implicit def newByteEncoder: Encoder[Byte]
- Definition Classes
- EncoderImplicits
- Since
1.6.0
- implicit def newDateEncoder: Encoder[Date]
- Definition Classes
- EncoderImplicits
- Since
2.2.0
- implicit val newDoubleArrayEncoder: Encoder[Array[Double]]
- Definition Classes
- EncoderImplicits
- Since
1.6.1
- implicit def newDoubleEncoder: Encoder[Double]
- Definition Classes
- EncoderImplicits
- Since
1.6.0
- implicit def newDurationEncoder: Encoder[Duration]
- Definition Classes
- EncoderImplicits
- Since
3.2.0
- implicit val newFloatArrayEncoder: Encoder[Array[Float]]
- Definition Classes
- EncoderImplicits
- Since
1.6.1
- implicit def newFloatEncoder: Encoder[Float]
- Definition Classes
- EncoderImplicits
- Since
1.6.0
- implicit def newInstantEncoder: Encoder[Instant]
- Definition Classes
- EncoderImplicits
- Since
3.0.0
- implicit val newIntArrayEncoder: Encoder[Array[Int]]
- Definition Classes
- EncoderImplicits
- Since
1.6.1
- implicit def newIntEncoder: Encoder[Int]
- Definition Classes
- EncoderImplicits
- Since
1.6.0
- implicit def newJavaDecimalEncoder: Encoder[BigDecimal]
- Definition Classes
- EncoderImplicits
- Since
2.2.0
- implicit def newJavaEnumEncoder[A <: Enum[_]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): Encoder[A]
- Definition Classes
- EncoderImplicits
- Since
3.2.0
- implicit def newLocalDateEncoder: Encoder[LocalDate]
- Definition Classes
- EncoderImplicits
- Since
3.0.0
- implicit def newLocalDateTimeEncoder: Encoder[LocalDateTime]
- Definition Classes
- EncoderImplicits
- Since
3.4.0
- implicit val newLongArrayEncoder: Encoder[Array[Long]]
- Definition Classes
- EncoderImplicits
- Since
1.6.1
- implicit def newLongEncoder: Encoder[Long]
- Definition Classes
- EncoderImplicits
- Since
1.6.0
- implicit def newMapEncoder[T <: Map[_, _]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]
- Definition Classes
- EncoderImplicits
- Since
2.3.0
- implicit def newPeriodEncoder: Encoder[Period]
- Definition Classes
- EncoderImplicits
- Since
3.2.0
- implicit def newProductArrayEncoder[A <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): Encoder[Array[A]]
- Definition Classes
- EncoderImplicits
- Since
1.6.1
- implicit def newProductEncoder[T <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]
- Definition Classes
- LowPrioritySQLImplicits
- Since
1.6.0
- implicit def newScalaDecimalEncoder: Encoder[BigDecimal]
- Definition Classes
- EncoderImplicits
- Since
2.2.0
- implicit def newSequenceEncoder[T <: Seq[_]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]
- Definition Classes
- EncoderImplicits
- Since
2.2.0
- implicit def newSetEncoder[T <: Set[_]](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[T]): Encoder[T]
Notice that we serialize
Set
to Catalyst array.Notice that we serialize
Set
to Catalyst array. The set property is only kept when manipulating the domain objects. The serialization format doesn't keep the set property. When we have a Catalyst array which contains duplicated elements and convert it toDataset[Set[T]]
by using the encoder, the elements will be de-duplicated.- Definition Classes
- EncoderImplicits
- Since
2.3.0
- implicit val newShortArrayEncoder: Encoder[Array[Short]]
- Definition Classes
- EncoderImplicits
- Since
1.6.1
- implicit def newShortEncoder: Encoder[Short]
- Definition Classes
- EncoderImplicits
- Since
1.6.0
- implicit val newStringArrayEncoder: Encoder[Array[String]]
- Definition Classes
- EncoderImplicits
- Since
1.6.1
- implicit def newStringEncoder: Encoder[String]
- Definition Classes
- EncoderImplicits
- Since
1.6.0
- implicit def newTimeStampEncoder: Encoder[Timestamp]
- Definition Classes
- EncoderImplicits
- Since
2.2.0
- final def notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @IntrinsicCandidate() @native()
- final def notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @IntrinsicCandidate() @native()
- implicit def symbolToColumn(s: Symbol): ColumnName
An implicit conversion that turns a Scala
Symbol
into a org.apache.spark.sql.Column.An implicit conversion that turns a Scala
Symbol
into a org.apache.spark.sql.Column.- Since
1.3.0
- final def synchronized[T0](arg0: => T0): T0
- Definition Classes
- AnyRef
- def toString(): String
- Definition Classes
- AnyRef → Any
- final def wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- final def wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException]) @native()
- final def wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
Deprecated Value Members
- def finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.Throwable]) @Deprecated
- Deprecated
(Since version 9)
- val newBooleanSeqEncoder: Encoder[Seq[Boolean]]
- Definition Classes
- EncoderImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- val newByteSeqEncoder: Encoder[Seq[Byte]]
- Definition Classes
- EncoderImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- val newDoubleSeqEncoder: Encoder[Seq[Double]]
- Definition Classes
- EncoderImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- val newFloatSeqEncoder: Encoder[Seq[Float]]
- Definition Classes
- EncoderImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- val newIntSeqEncoder: Encoder[Seq[Int]]
- Definition Classes
- EncoderImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- val newLongSeqEncoder: Encoder[Seq[Long]]
- Definition Classes
- EncoderImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- def newProductSeqEncoder[A <: Product](implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): Encoder[Seq[A]]
- Definition Classes
- EncoderImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- val newShortSeqEncoder: Encoder[Seq[Short]]
- Definition Classes
- EncoderImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1
- val newStringSeqEncoder: Encoder[Seq[String]]
- Definition Classes
- EncoderImplicits
- Annotations
- @deprecated
- Deprecated
(Since version 2.2.0) Use newSequenceEncoder instead
- Since
1.6.1