Package org.apache.sysds.runtime.io

Class IOUtilFunctions

java.lang.Object
org.apache.sysds.runtime.io.IOUtilFunctions
public class IOUtilFunctions extends Object

  • Field Details

  • Constructor Details

    • IOUtilFunctions

      public IOUtilFunctions()

  • Method Details

    • getFileSystem

      public static org.apache.hadoop.fs.FileSystem getFileSystem(String fname) throws IOException
      Throws:
      IOException

    • getFileSystem

      public static org.apache.hadoop.fs.FileSystem getFileSystem(org.apache.hadoop.fs.Path fname) throws IOException
      Throws:
      IOException

    • getFileSystem

      public static org.apache.hadoop.fs.FileSystem getFileSystem(org.apache.hadoop.conf.Configuration conf) throws IOException
      Throws:
      IOException

    • getFileSystem

      public static org.apache.hadoop.fs.FileSystem getFileSystem(org.apache.hadoop.fs.Path fname, org.apache.hadoop.conf.Configuration conf) throws IOException
      Throws:
      IOException

    • isSameFileScheme

      public static boolean isSameFileScheme(org.apache.hadoop.fs.Path path1, org.apache.hadoop.fs.Path path2)

    • isObjectStoreFileScheme

      public static boolean isObjectStoreFileScheme(org.apache.hadoop.fs.Path path)

    • getPartFileName

      public static String getPartFileName(int pos)

    • closeSilently

      public static void closeSilently(Closeable io)

    • closeSilently

      public static void closeSilently(org.apache.hadoop.mapred.RecordReader<?,?> rr)

    • checkAndRaiseErrorCSVEmptyField

      public static void checkAndRaiseErrorCSVEmptyField(String row, boolean fill, boolean emptyFound) throws IOException
      Throws:
      IOException

    • checkAndRaiseErrorCSVNumColumns

      public static void checkAndRaiseErrorCSVNumColumns(org.apache.hadoop.mapred.InputSplit split, String line, String[] parts, long ncol) throws IOException
      Throws:
      IOException

    • checkAndRaiseErrorCSVNumColumns

      public static void checkAndRaiseErrorCSVNumColumns(String src, String line, String[] parts, long ncol) throws IOException
      Throws:
      IOException

    • split

      public static String[] split(String str, String delim)
      Splits a string by a specified delimiter into all tokens, including empty. NOTE: This method is meant as a faster drop-in replacement of the regular string split.
      Parameters:
      str - string to split
      delim - delimiter
      Returns:
      string array

    • splitCSV

      public static String[] splitCSV(String str, String delim)
      Splits a string by a specified delimiter into all tokens, including empty while respecting the rules for quotes and escapes defined in RFC4180, with robustness for various special cases.
      Parameters:
      str - string to split
      delim - delimiter
      Returns:
      string array of tokens

    • splitCSV

      public static String[] splitCSV(String str, String delim, int clen)

    • splitCSV

      public static String[] splitCSV(String str, String delim, String[] cache)
      Splits a string by a specified delimiter into all tokens, including empty while respecting the rules for quotes and escapes defined in RFC4180, with robustness for various special cases.
      Parameters:
      str - string to split
      delim - delimiter
      cache - cachedReturnArray
      Returns:
      string array of tokens

    • getTo

      public static int getTo(String str, int from, String delim, int len, int dLen)
      Get next index of substring after delim, while the string can contain Quotation marks
      Parameters:
      str - The string to get the index from
      from - The index to start searching from
      delim - The delimiter to find
      len - The length of the str string argument
      dLen - The length of the delimiter string
      Returns:
      The next index.

    • trim

      public static String trim(String str)

    • trim

      public static String trim(String str, int len)
      Caller must have a string of at least 1 character length.
      Parameters:
      str - string to trim
      len - length of string
      Returns:
      the trimmed string.

    • splitCSV

      public static String[] splitCSV(String str, String delim, String[] tokens, Set<String> naStrings)
      Splits a string by a specified delimiter into all tokens, including empty while respecting the rules for quotes and escapes defined in RFC4180, with robustness for various special cases.
      Parameters:
      str - string to split
      delim - delimiter
      tokens - array for tokens, length needs to match the number of tokens
      naStrings - the strings to map to null value.
      Returns:
      string array of tokens

    • countTokensCSV

      public static int countTokensCSV(String str, String delim)
      Counts the number of tokens defined by the given delimiter, respecting the rules for quotes and escapes defined in RFC4180, with robustness for various special cases.
      Parameters:
      str - string to split
      delim - delimiter
      Returns:
      number of tokens split by the given delimiter

    • splitByFirst

      public static String[] splitByFirst(String str, String delim)

    • readAndParseMatrixMarketHeader

      public static FileFormatPropertiesMM readAndParseMatrixMarketHeader(String filename) throws DMLRuntimeException
      Throws:
      DMLRuntimeException

    • readMatrixMarketHeader

      public static String[] readMatrixMarketHeader(String filename)

    • countNnz

      public static int countNnz(String[] cols)
      Returns the number of non-zero entries but avoids the expensive string to double parsing. This function is guaranteed to never underestimate.
      Parameters:
      cols - string array
      Returns:
      number of non-zeros

    • countNnz

      public static int countNnz(String[] cols, int pos, int len)
      Returns the number of non-zero entries but avoids the expensive string to double parsing. This function is guaranteed to never underestimate.
      Parameters:
      cols - string array
      pos - starting array index
      len - ending array index
      Returns:
      number of non-zeros

    • getUTFSize

      public static int getUTFSize(String value)
      Returns the serialized size in bytes of the given string value, following the modified UTF-8 specification as used by Java's DataInput/DataOutput. see java docs: docs/api/java/io/DataInput.html#modified-utf-8
      Parameters:
      value - string value
      Returns:
      string size for modified UTF-8 specification

    • toInputStream

      public static InputStream toInputStream(String input)

    • toString

      public static String toString(InputStream input) throws IOException
      Throws:
      IOException

    • sortInputSplits

      public static org.apache.hadoop.mapred.InputSplit[] sortInputSplits(org.apache.hadoop.mapred.InputSplit[] splits)

    • countNumColumnsCSV

      public static int countNumColumnsCSV(org.apache.hadoop.mapred.InputSplit[] splits, org.apache.hadoop.mapred.InputFormat informat, org.apache.hadoop.mapred.JobConf job, String delim) throws IOException
      Counts the number of columns in a given collection of csv file splits. This primitive aborts if a row with more than 0 columns is found and hence is robust against empty file splits etc.
      Parameters:
      splits - input splits
      informat - input format
      job - job configruation
      delim - delimiter
      Returns:
      the number of columns in the collection of csv file splits
      Throws:
      IOException - if IOException occurs

    • getSequenceFilePaths

      public static org.apache.hadoop.fs.Path[] getSequenceFilePaths(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path file) throws IOException
      Throws:
      IOException

    • getMetadataFilePaths

      public static org.apache.hadoop.fs.Path[] getMetadataFilePaths(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path file) throws IOException
      Throws:
      IOException

    • deleteCrcFilesFromLocalFileSystem

      public static void deleteCrcFilesFromLocalFileSystem(org.apache.hadoop.mapred.JobConf job, org.apache.hadoop.fs.Path path) throws IOException
      Throws:
      IOException

    • deleteCrcFilesFromLocalFileSystem

      public static void deleteCrcFilesFromLocalFileSystem(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path) throws IOException
      Delete the CRC files from the local file system associated with a particular file and its metadata file.
      Parameters:
      fs - the file system
      path - the path to a file
      Throws:
      IOException - thrown if error occurred attempting to delete crc files

    • baToShort

      public static int baToShort(byte[] ba, int off)

    • baToInt

      public static int baToInt(byte[] ba, int off)

    • baToLong

      public static long baToLong(byte[] ba, int off)

    • shortToBa

      public static void shortToBa(int val, byte[] ba, int off)

    • intToBa

      public static void intToBa(int val, byte[] ba, int off)

    • longToBa

      public static void longToBa(long val, byte[] ba, int off)

    • getBytes

      public static byte[] getBytes(ByteBuffer buff)

    • get

      public static <T> T get(Future<T> in)

    • isFileCPReadable

      public static boolean isFileCPReadable(String path)

    • getSeqWriter

      public static org.apache.hadoop.io.SequenceFile.Writer getSeqWriter(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration job, int replication) throws IOException
      Throws:
      IOException

    • getSeqWriterFrame

      public static org.apache.hadoop.io.SequenceFile.Writer getSeqWriterFrame(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration job, int replication) throws IOException
      Throws:
      IOException

    • getSeqWriterArray

      public static org.apache.hadoop.io.SequenceFile.Writer getSeqWriterArray(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration job, int replication) throws IOException
      Throws:
      IOException

    • getSeqWriterTensor

      public static org.apache.hadoop.io.SequenceFile.Writer getSeqWriterTensor(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration job, int replication) throws IOException
      Throws:
      IOException

    • getSeqWriterCell

      public static org.apache.hadoop.io.SequenceFile.Writer getSeqWriterCell(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration job, int replication) throws IOException
      Throws:
      IOException

    • getCompressionEncodingType

      public static org.apache.hadoop.io.SequenceFile.CompressionType getCompressionEncodingType()

    • getCompressionCodec

      public static org.apache.hadoop.io.compress.CompressionCodec getCompressionCodec()