c
io.smartdatalake.workflow.dataobject
SnowflakeTableDataObject
Companion object SnowflakeTableDataObject
case class SnowflakeTableDataObject(id: DataObjectId, schemaMin: Option[StructType] = None, table: Table, saveMode: SDLSaveMode = SDLSaveMode.Overwrite, connectionId: ConnectionId, comment: Option[String], metadata: Option[DataObjectMetadata] = None)(implicit instanceRegistry: InstanceRegistry) extends TransactionalSparkTableDataObject with Product with Serializable
DataObject of type SnowflakeTableDataObject. Provides details to access Snowflake tables via an action
- id
unique name of this data object
- schemaMin
An optional, minimal schema that this DataObject must have to pass schema validation on reading and writing.
- table
Snowflake table to be written by this output
- saveMode
spark SDLSaveMode to use when writing files, default is "overwrite"
- connectionId
The SnowflakeTableConnection to use for the table
- comment
An optional comment to add to the table after writing a DataFrame to it
- metadata
meta data
Linear Supertypes
Ordering
- Alphabetic
- By Inheritance
Inherited
- SnowflakeTableDataObject
- Serializable
- Serializable
- Product
- Equals
- TransactionalSparkTableDataObject
- CanWriteDataFrame
- TableDataObject
- SchemaValidation
- CanCreateDataFrame
- DataObject
- AtlasExportable
- SmartDataLakeLogger
- ParsableFromConfig
- SdlConfigObject
- AnyRef
- Any
- Hide All
- Show All
Visibility
- Public
- All
Instance Constructors
-
new
SnowflakeTableDataObject(id: DataObjectId, schemaMin: Option[StructType] = None, table: Table, saveMode: SDLSaveMode = SDLSaveMode.Overwrite, connectionId: ConnectionId, comment: Option[String], metadata: Option[DataObjectMetadata] = None)(implicit instanceRegistry: InstanceRegistry)
- id
unique name of this data object
- schemaMin
An optional, minimal schema that this DataObject must have to pass schema validation on reading and writing.
- table
Snowflake table to be written by this output
- saveMode
spark SDLSaveMode to use when writing files, default is "overwrite"
- connectionId
The SnowflakeTableConnection to use for the table
- comment
An optional comment to add to the table after writing a DataFrame to it
- metadata
meta data
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
addFieldIfNotExisting(writeSchema: StructType, colName: String, dataType: DataType): StructType
- Attributes
- protected
- Definition Classes
- CanCreateDataFrame
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
def
atlasName: String
- Definition Classes
- TableDataObject → DataObject → AtlasExportable
-
def
atlasQualifiedName(prefix: String): String
- Definition Classes
- TableDataObject → AtlasExportable
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
- val comment: Option[String]
- val connectionId: ConnectionId
-
def
createReadSchema(writeSchema: StructType)(implicit session: SparkSession): StructType
- Definition Classes
- CanCreateDataFrame
-
def
dropTable(implicit session: SparkSession): Unit
- Definition Classes
- SnowflakeTableDataObject → TableDataObject
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
factory: FromConfigFactory[DataObject]
- Definition Classes
- SnowflakeTableDataObject → ParsableFromConfig
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
getConnection[T <: Connection](connectionId: ConnectionId)(implicit registry: InstanceRegistry, ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T
- Attributes
- protected
- Definition Classes
- DataObject
-
def
getConnectionReg[T <: Connection](connectionId: ConnectionId, registry: InstanceRegistry)(implicit ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T
- Attributes
- protected
- Definition Classes
- DataObject
-
def
getDataFrame(partitionValues: Seq[PartitionValues] = Seq())(implicit session: SparkSession, context: ActionPipelineContext): DataFrame
- Definition Classes
- SnowflakeTableDataObject → CanCreateDataFrame
-
def
getPKduplicates(implicit session: SparkSession, context: ActionPipelineContext): DataFrame
- Definition Classes
- TableDataObject
-
def
getPKnulls(implicit session: SparkSession, context: ActionPipelineContext): DataFrame
- Definition Classes
- TableDataObject
-
def
getPKviolators(implicit session: SparkSession, context: ActionPipelineContext): DataFrame
- Definition Classes
- TableDataObject
-
def
housekeepingMode: Option[HousekeepingMode]
- Definition Classes
- DataObject
-
val
id: DataObjectId
- Definition Classes
- SnowflakeTableDataObject → DataObject → SdlConfigObject
-
def
init(df: DataFrame, partitionValues: Seq[PartitionValues], saveModeOptions: Option[SaveModeOptions])(implicit session: SparkSession, context: ActionPipelineContext): Unit
- Definition Classes
- CanWriteDataFrame
- implicit val instanceRegistry: InstanceRegistry
-
def
isDbExisting(implicit session: SparkSession): Boolean
- Definition Classes
- SnowflakeTableDataObject → TableDataObject
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
def
isPKcandidateKey(implicit session: SparkSession, context: ActionPipelineContext): Boolean
- Definition Classes
- TableDataObject
-
def
isTableExisting(implicit session: SparkSession): Boolean
- Definition Classes
- SnowflakeTableDataObject → TableDataObject
-
lazy val
logger: Logger
- Attributes
- protected
- Definition Classes
- SmartDataLakeLogger
- Annotations
- @transient()
-
val
metadata: Option[DataObjectMetadata]
- Definition Classes
- SnowflakeTableDataObject → DataObject
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- val saveMode: SDLSaveMode
-
val
schemaMin: Option[StructType]
- Definition Classes
- SnowflakeTableDataObject → SchemaValidation
-
def
streamingOptions: Map[String, String]
- Definition Classes
- CanWriteDataFrame
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
var
table: Table
- Definition Classes
- SnowflakeTableDataObject → TableDataObject
-
val
tableSchema: StructType
- Definition Classes
- TableDataObject
-
def
toStringShort: String
- Definition Classes
- DataObject
-
def
validateSchema(df: DataFrame, schemaExpected: StructType, role: String): Unit
- Definition Classes
- SchemaValidation
-
def
validateSchemaMin(df: DataFrame, role: String): Unit
- Definition Classes
- SchemaValidation
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
writeDataFrame(df: DataFrame, createTableOnly: Boolean, partitionValues: Seq[PartitionValues], saveModeOptions: Option[SaveModeOptions])(implicit session: SparkSession): Unit
Writes DataFrame to Snowflake Snowflake does not support explicit partitions, so any passed partition values are ignored
-
def
writeDataFrame(df: DataFrame, partitionValues: Seq[PartitionValues] = Seq(), isRecursiveInput: Boolean = false, saveModeOptions: Option[SaveModeOptions] = None)(implicit session: SparkSession, context: ActionPipelineContext): Unit
- Definition Classes
- SnowflakeTableDataObject → CanWriteDataFrame
-
def
writeStreamingDataFrame(df: DataFrame, trigger: Trigger, options: Map[String, String], checkpointLocation: String, queryName: String, outputMode: OutputMode, saveModeOptions: Option[SaveModeOptions])(implicit session: SparkSession, context: ActionPipelineContext): StreamingQuery
- Definition Classes
- CanWriteDataFrame