public abstract class BaseExternalLogStore
extends io.delta.storage.HadoopFileSystemLogStore
LogStore implementation for cloud stores (e.g. Amazon S3) that do not provide
mutual exclusion.
This implementation depends on child methods, particularly `putExternalEntry`, to provide the mutual exclusion that the cloud store is lacking.
| Constructor and Description |
|---|
BaseExternalLogStore(org.apache.hadoop.conf.Configuration hadoopConf) |
| Modifier and Type | Method and Description |
|---|---|
protected java.lang.String |
createTemporaryPath(org.apache.hadoop.fs.Path path)
Generate temporary path for TransactionLog.
|
protected void |
fixDeltaLogCopyTempFile(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path src,
org.apache.hadoop.fs.Path dst)
Wrapper for `copyFile`, called by the `fixDeltaLog` method.
|
protected void |
fixDeltaLogPutCompleteDbEntry(ExternalCommitEntry entry)
Wrapper for `putExternalEntry`, called by the `fixDeltaLog` method.
|
protected abstract java.util.Optional<ExternalCommitEntry> |
getExternalEntry(java.lang.String tablePath,
java.lang.String fileName)
Return external store entry corresponding to delta log file with given `tablePath` and
`fileName`, or `Optional.empty()` if it doesn't exist.
|
protected abstract java.util.Optional<ExternalCommitEntry> |
getLatestExternalEntry(org.apache.hadoop.fs.Path tablePath)
Return the latest external store entry corresponding to the delta log for given `tablePath`,
or `Optional.empty()` if it doesn't exist.
|
protected org.apache.hadoop.fs.Path |
getTablePath(org.apache.hadoop.fs.Path path)
Returns the base table path for a given Delta log entry located in
e.g.
|
java.lang.Boolean |
isPartialWriteVisible(org.apache.hadoop.fs.Path path,
org.apache.hadoop.conf.Configuration hadoopConf) |
java.util.Iterator<org.apache.hadoop.fs.FileStatus> |
listFrom(org.apache.hadoop.fs.Path path,
org.apache.hadoop.conf.Configuration hadoopConf)
First checks if there is any incomplete entry in the external store.
|
protected abstract void |
putExternalEntry(ExternalCommitEntry entry,
boolean overwrite)
Write to external store in exclusive way.
|
void |
write(org.apache.hadoop.fs.Path path,
java.util.Iterator<java.lang.String> actions,
java.lang.Boolean overwrite,
org.apache.hadoop.conf.Configuration hadoopConf)
If overwrite=true, then write normally without any interaction with external store.
|
protected void |
writeActions(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
java.util.Iterator<java.lang.String> actions)
Write file with actions under a specific path.
|
protected void |
writeCopyTempFile(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path src,
org.apache.hadoop.fs.Path dst)
Wrapper for `copyFile`, called by the `write` method.
|
protected void |
writePutCompleteDbEntry(ExternalCommitEntry entry)
Wrapper for `putExternalEntry`, called by the `write` method.
|
public BaseExternalLogStore(org.apache.hadoop.conf.Configuration hadoopConf)
public java.util.Iterator<org.apache.hadoop.fs.FileStatus> listFrom(org.apache.hadoop.fs.Path path,
org.apache.hadoop.conf.Configuration hadoopConf)
throws java.io.IOException
listFrom in class io.delta.storage.HadoopFileSystemLogStorejava.io.IOExceptionpublic void write(org.apache.hadoop.fs.Path path,
java.util.Iterator<java.lang.String> actions,
java.lang.Boolean overwrite,
org.apache.hadoop.conf.Configuration hadoopConf)
throws java.io.IOException
write in class io.delta.storage.LogStorejava.io.IOExceptionpublic java.lang.Boolean isPartialWriteVisible(org.apache.hadoop.fs.Path path,
org.apache.hadoop.conf.Configuration hadoopConf)
isPartialWriteVisible in class io.delta.storage.LogStoreprotected void writeActions(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
java.util.Iterator<java.lang.String> actions)
throws java.io.IOException
java.io.IOExceptionprotected java.lang.String createTemporaryPath(org.apache.hadoop.fs.Path path)
protected org.apache.hadoop.fs.Path getTablePath(org.apache.hadoop.fs.Path path)
protected abstract void putExternalEntry(ExternalCommitEntry entry, boolean overwrite) throws java.io.IOException
java.nio.file.FileAlreadyExistsException - if path exists in cache and `overwrite` is
falsejava.io.IOExceptionprotected abstract java.util.Optional<ExternalCommitEntry> getExternalEntry(java.lang.String tablePath, java.lang.String fileName) throws java.io.IOException
java.io.IOExceptionprotected abstract java.util.Optional<ExternalCommitEntry> getLatestExternalEntry(org.apache.hadoop.fs.Path tablePath) throws java.io.IOException
tablePath - TODOjava.io.IOExceptionprotected void writeCopyTempFile(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path src,
org.apache.hadoop.fs.Path dst)
throws java.io.IOException
java.io.IOExceptionprotected void writePutCompleteDbEntry(ExternalCommitEntry entry) throws java.io.IOException
java.io.IOExceptionprotected void fixDeltaLogCopyTempFile(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path src,
org.apache.hadoop.fs.Path dst)
throws java.io.IOException
java.io.IOExceptionprotected void fixDeltaLogPutCompleteDbEntry(ExternalCommitEntry entry) throws java.io.IOException
java.io.IOException