Package org.apache.sysds.resource.cost

Class SparkCostUtils

java.lang.Object

org.apache.sysds.resource.cost.SparkCostUtils

public class SparkCostUtils
extends Object

Constructor Summary

Constructors

Constructor	Description
SparkCostUtils()

Method Summary

Modifier and Type	Method	Description
static void	assignOutputRDDStats(SPInstruction inst, VarStats output, VarStats... inputs)
static double	getAggUnaryInstTime(UnarySPInstruction inst, VarStats input, VarStats output, IOCostUtils.IOMetrics executorMetrics)
static double	getAppendInstTime(AppendSPInstruction inst, VarStats input1, VarStats input2, VarStats output, IOCostUtils.IOMetrics driverMetrics, IOCostUtils.IOMetrics executorMetrics)
static double	getBinaryInstTime(SPInstruction inst, VarStats input1, VarStats input2, VarStats output, IOCostUtils.IOMetrics driverMetrics, IOCostUtils.IOMetrics executorMetrics)
static double	getCastInstTime(CastSPInstruction inst, VarStats input, VarStats output, IOCostUtils.IOMetrics executorMetrics)
static double	getCentralMomentInstTime(CentralMomentSPInstruction inst, VarStats input, VarStats weights, VarStats output, IOCostUtils.IOMetrics executorMetrics)
static double	getCPUTime(long nflop, int numPartitions, IOCostUtils.IOMetrics executorMetrics, RDDStats output, RDDStats... inputs)	Computes an estimate for the time needed by the CPU to execute (including memory access) an instruction by providing number of floating operations.
static double	getCtableInstTime(CtableSPInstruction tableInst, VarStats input1, VarStats input2, VarStats input3, VarStats output, IOCostUtils.IOMetrics executorMetrics)
static double	getIndexingInstTime(IndexingSPInstruction inst, VarStats input1, VarStats input2, VarStats output, IOCostUtils.IOMetrics driverMetrics, IOCostUtils.IOMetrics executorMetrics)
static double	getMatMulChainInstTime(MapmmChainSPInstruction inst, VarStats input1, VarStats input2, VarStats input3, VarStats output, IOCostUtils.IOMetrics driverMetrics, IOCostUtils.IOMetrics executorMetrics)
static double	getMatMulInstTime(BinarySPInstruction inst, VarStats input1, VarStats input2, VarStats output, IOCostUtils.IOMetrics driverMetrics, IOCostUtils.IOMetrics executorMetrics)
static double	getParameterizedBuiltinInstTime(ParameterizedBuiltinSPInstruction paramInst, VarStats input1, VarStats input2, VarStats output, IOCostUtils.IOMetrics driverMetrics, IOCostUtils.IOMetrics executorMetrics)
static double	getQSortInstTime(QuantileSortSPInstruction inst, VarStats input, VarStats weights, VarStats output, IOCostUtils.IOMetrics executorMetrics)
static double	getQuaternaryInstTime(QuaternarySPInstruction quatInst, VarStats input1, VarStats input2, VarStats input3, VarStats output, IOCostUtils.IOMetrics driverMetrics, IOCostUtils.IOMetrics executorMetrics)
static double	getRandInstTime(String opcode, int randType, VarStats output, IOCostUtils.IOMetrics executorMetrics)
static double	getReblockInstTime(String opcode, VarStats input, VarStats output, IOCostUtils.IOMetrics executorMetrics)
static double	getReorgInstTime(UnarySPInstruction inst, VarStats input, VarStats output, IOCostUtils.IOMetrics executorMetrics)
static double	getTernaryInstTime(TernarySPInstruction tInst, VarStats input1, VarStats input2, VarStats input3, VarStats output, IOCostUtils.IOMetrics executorMetrics)
static double	getTSMMInstTime(UnarySPInstruction inst, VarStats input, VarStats output, IOCostUtils.IOMetrics driverMetrics, IOCostUtils.IOMetrics executorMetrics)
static double	getUnaryInstTime(String opcode, VarStats input, VarStats output, IOCostUtils.IOMetrics executorMetrics)

Methods inherited from class java.lang.Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Details

SparkCostUtils

public SparkCostUtils()

Method Details

getReblockInstTime

public static double getReblockInstTime(String opcode,
 VarStats input,
 VarStats output,
 IOCostUtils.IOMetrics executorMetrics)

getRandInstTime

public static double getRandInstTime(String opcode,
 int randType,
 VarStats output,
 IOCostUtils.IOMetrics executorMetrics)

getUnaryInstTime

public static double getUnaryInstTime(String opcode,
 VarStats input,
 VarStats output,
 IOCostUtils.IOMetrics executorMetrics)

getAggUnaryInstTime

public static double getAggUnaryInstTime(UnarySPInstruction inst,
 VarStats input,
 VarStats output,
 IOCostUtils.IOMetrics executorMetrics)

getIndexingInstTime

public static double getIndexingInstTime(IndexingSPInstruction inst,
 VarStats input1,
 VarStats input2,
 VarStats output,
 IOCostUtils.IOMetrics driverMetrics,
 IOCostUtils.IOMetrics executorMetrics)

getBinaryInstTime

public static double getBinaryInstTime(SPInstruction inst,
 VarStats input1,
 VarStats input2,
 VarStats output,
 IOCostUtils.IOMetrics driverMetrics,
 IOCostUtils.IOMetrics executorMetrics)

getAppendInstTime

public static double getAppendInstTime(AppendSPInstruction inst,
 VarStats input1,
 VarStats input2,
 VarStats output,
 IOCostUtils.IOMetrics driverMetrics,
 IOCostUtils.IOMetrics executorMetrics)

getReorgInstTime

public static double getReorgInstTime(UnarySPInstruction inst,
 VarStats input,
 VarStats output,
 IOCostUtils.IOMetrics executorMetrics)

getTSMMInstTime

public static double getTSMMInstTime(UnarySPInstruction inst,
 VarStats input,
 VarStats output,
 IOCostUtils.IOMetrics driverMetrics,
 IOCostUtils.IOMetrics executorMetrics)

getCentralMomentInstTime

public static double getCentralMomentInstTime(CentralMomentSPInstruction inst,
 VarStats input,
 VarStats weights,
 VarStats output,
 IOCostUtils.IOMetrics executorMetrics)

getCastInstTime

public static double getCastInstTime(CastSPInstruction inst,
 VarStats input,
 VarStats output,
 IOCostUtils.IOMetrics executorMetrics)

getQSortInstTime

public static double getQSortInstTime(QuantileSortSPInstruction inst,
 VarStats input,
 VarStats weights,
 VarStats output,
 IOCostUtils.IOMetrics executorMetrics)

getMatMulInstTime

public static double getMatMulInstTime(BinarySPInstruction inst,
 VarStats input1,
 VarStats input2,
 VarStats output,
 IOCostUtils.IOMetrics driverMetrics,
 IOCostUtils.IOMetrics executorMetrics)

getMatMulChainInstTime

public static double getMatMulChainInstTime(MapmmChainSPInstruction inst,
 VarStats input1,
 VarStats input2,
 VarStats input3,
 VarStats output,
 IOCostUtils.IOMetrics driverMetrics,
 IOCostUtils.IOMetrics executorMetrics)

getCtableInstTime

public static double getCtableInstTime(CtableSPInstruction tableInst,
 VarStats input1,
 VarStats input2,
 VarStats input3,
 VarStats output,
 IOCostUtils.IOMetrics executorMetrics)

getParameterizedBuiltinInstTime

public static double getParameterizedBuiltinInstTime(ParameterizedBuiltinSPInstruction paramInst,
 VarStats input1,
 VarStats input2,
 VarStats output,
 IOCostUtils.IOMetrics driverMetrics,
 IOCostUtils.IOMetrics executorMetrics)

getTernaryInstTime

public static double getTernaryInstTime(TernarySPInstruction tInst,
 VarStats input1,
 VarStats input2,
 VarStats input3,
 VarStats output,
 IOCostUtils.IOMetrics executorMetrics)

getQuaternaryInstTime

public static double getQuaternaryInstTime(QuaternarySPInstruction quatInst,
 VarStats input1,
 VarStats input2,
 VarStats input3,
 VarStats output,
 IOCostUtils.IOMetrics driverMetrics,
 IOCostUtils.IOMetrics executorMetrics)

getCPUTime

public static double getCPUTime(long nflop,
 int numPartitions,
 IOCostUtils.IOMetrics executorMetrics,
 RDDStats output,
 RDDStats... inputs)

Computes an estimate for the time needed by the CPU to execute (including memory access) an instruction by providing number of floating operations.

Parameters:

nflop - number FLOP to execute a target CPU operation

numPartitions - number partitions used to execute the target operation; not bound to any of the input/output statistics object to allow more flexibility depending on the corresponding instruction

executorMetrics - metrics for the executor utilized by the Spark cluster

output - statistics for the output variable

inputs - arrays of statistics for the output variable

Returns:

time estimate

assignOutputRDDStats

public static void assignOutputRDDStats(SPInstruction inst,
 VarStats output,
 VarStats... inputs)