From 8cfaa77db432efe29e18b67820dba72fcd7ad88d Mon Sep 17 00:00:00 2001 From: curryzjj Date: Fri, 11 Mar 2022 09:19:12 +0800 Subject: [PATCH 01/15] Add Collector --- affinity/src/main/c/Makefile | 5 ++- .../bolts/transactional/gs/GSBolt_ts.java | 1 - pom.xml | 43 +++++++++++++++++++ .../java/scheduler/collector/Collector.java | 43 +++++++++++++++++++ .../src/main/java/transaction/TxnManager.java | 17 ++++++++ .../main/java/execution/ExecutionManager.java | 2 + 6 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 state-engine/src/main/java/scheduler/collector/Collector.java diff --git a/affinity/src/main/c/Makefile b/affinity/src/main/c/Makefile index 58a06e53a..85e114c74 100755 --- a/affinity/src/main/c/Makefile +++ b/affinity/src/main/c/Makefile @@ -19,7 +19,8 @@ JNI_JAVA_SOURCES := $(patsubst %,$(TARGET_DIR)/%.class,$(subst .,/,$(JAVA_CLASSE JAVA_BUILD_DIR := $(TARGET_DIR) -JAVA_HOME ?= /usr/java/default +#JAVA_HOME ?= /usr/java/default +JAVA_HOME :=/Library/Java/JavaVirtualMachines/jdk1.8.0_191.jdk/Contents/Home JAVA_LIB := $(JAVA_HOME)/jre/lib JVM_SHARED_LIBS := -L$(JAVA_LIB)/amd64/server -L$(JAVA_LIB)/i386/server -L$(JAVA_LIB)/amd64/jrockit/ -L$(JAVA_LIB)/i386/jrockit/ -L$(JAVA_LIB)/ppc64le/server -L$(JAVA_LIB)/ppc64le/jrockit/ @@ -42,7 +43,7 @@ $(TARGET): $(JNI_HEADERS) $(JNI_SOURCES) $(JNI_HEADERS): $(JNI_JAVA_SOURCES) mkdir -p $(TARGET_DIR)/jni - javah -force -classpath $(JAVAH_CLASSPATH) -d $(WORKING_DIR) $(JAVA_CLASSES) + javac -h -force -classpath $(JAVAH_CLASSPATH) -d $(WORKING_DIR) $(JAVA_CLASSES) clean: rm $(TARGET) diff --git a/application/src/main/java/common/bolts/transactional/gs/GSBolt_ts.java b/application/src/main/java/common/bolts/transactional/gs/GSBolt_ts.java index 66fbdb6cf..c6fab73f4 100644 --- a/application/src/main/java/common/bolts/transactional/gs/GSBolt_ts.java +++ b/application/src/main/java/common/bolts/transactional/gs/GSBolt_ts.java @@ -141,7 +141,6 @@ public void execute(Tuple in) throws InterruptedException, DatabaseException, Br READ_POST(); } END_POST_TIME_MEASURE_ACC(thread_Id); - //all tuples in the holder is finished. microEvents.clear(); } diff --git a/pom.xml b/pom.xml index 2345b66af..2b3c23657 100644 --- a/pom.xml +++ b/pom.xml @@ -16,6 +16,49 @@ + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + org.apache.maven.plugins diff --git a/state-engine/src/main/java/scheduler/collector/Collector.java b/state-engine/src/main/java/scheduler/collector/Collector.java new file mode 100644 index 000000000..8a61054a8 --- /dev/null +++ b/state-engine/src/main/java/scheduler/collector/Collector.java @@ -0,0 +1,43 @@ +package scheduler.collector; + +import java.util.HashMap; + +public class Collector { + /* + Num of TD,LD,PD + */ + private HashMap Num_of_dependency; + /* + Vertex Degree Distribution + */ + private double VDD; + /* + Is Cyclic Dependency + */ + private boolean IsCyclicDependency; + /* + Computation Complexity, if high return true + */ + private boolean IsComputationComplexity; + /* + Rate of Aborting Vertexes + */ + private double R_of_A; + + /** + * Implement the decision tree + * Return the decision of the scheduler switching + * @return + */ + public int getDecision(){ + return 0; + } + + /** + * Init the collector after each switch + */ + private void InitCollector(){ + + } + +} diff --git a/state-engine/src/main/java/transaction/TxnManager.java b/state-engine/src/main/java/transaction/TxnManager.java index 9cba8d714..34b1d7ee3 100644 --- a/state-engine/src/main/java/transaction/TxnManager.java +++ b/state-engine/src/main/java/transaction/TxnManager.java @@ -61,4 +61,21 @@ public static void CreateScheduler(String schedulerType, int threadCount, int nu throw new UnsupportedOperationException("unsupported scheduler type: " + schedulerType); } } + + /** + * Switch scheduler every punctuation + * When the workload changes and the scheduler is no longer applicable + * @return + */ + public boolean SwitchScheduler(){ + //TODO: implement the scheduler switching + return true; + } + + /** + * Configure the bottom line for triggering scheduled switching in Collector + */ + public static void setBottomLine(){ + + } } diff --git a/stream-engine/src/main/java/execution/ExecutionManager.java b/stream-engine/src/main/java/execution/ExecutionManager.java index 69789ad5e..bc839cdd7 100644 --- a/stream-engine/src/main/java/execution/ExecutionManager.java +++ b/stream-engine/src/main/java/execution/ExecutionManager.java @@ -67,6 +67,8 @@ public void distributeTasks(Configuration conf, String schedulerType = conf.getString("scheduler"); int app = conf.getInt("app"); TxnManager.CreateScheduler(schedulerType, totalThread, numberOfStates, app); + //TODO: Configure the bottom line for triggering scheduler switching in Collector + TxnManager.setBottomLine(); } } executorThread thread = null; From fbe53b888ff230b7adf9290eddf171dbb48f1d0c Mon Sep 17 00:00:00 2001 From: curryzjj Date: Sat, 12 Mar 2022 22:17:48 +0800 Subject: [PATCH 02/15] Add application Bolt(TP,OB) finish dynamic scheduler --- .../bolts/transactional/ob/OBBolt_ts_s.java | 159 ++++++++++++++++++ .../bolts/transactional/tp/TPBolt_ts.java | 3 +- .../bolts/transactional/tp/TPBolt_ts_s.java | 117 +++++++++++++ common/src/main/java/common/Runner.java | 15 ++ .../java/scheduler/collector/Collector.java | 51 +++++- .../struct/og/TaskPrecedenceGraph.java | 14 +- .../struct/op/TaskPrecedenceGraph.java | 14 +- .../src/main/java/transaction/TxnManager.java | 60 ++++++- .../main/java/execution/ExecutionManager.java | 12 +- 9 files changed, 427 insertions(+), 18 deletions(-) create mode 100644 application/src/main/java/common/bolts/transactional/ob/OBBolt_ts_s.java create mode 100644 application/src/main/java/common/bolts/transactional/tp/TPBolt_ts_s.java diff --git a/application/src/main/java/common/bolts/transactional/ob/OBBolt_ts_s.java b/application/src/main/java/common/bolts/transactional/ob/OBBolt_ts_s.java new file mode 100644 index 000000000..044b5fec1 --- /dev/null +++ b/application/src/main/java/common/bolts/transactional/ob/OBBolt_ts_s.java @@ -0,0 +1,159 @@ +package common.bolts.transactional.ob; + +import combo.SINKCombo; +import common.param.TxnEvent; +import common.param.ob.AlertEvent; +import common.param.ob.BuyingEvent; +import common.param.ob.ToppingEvent; +import components.context.TopologyContext; +import db.DatabaseException; +import execution.ExecutionGraph; +import execution.runtime.collector.OutputCollector; +import execution.runtime.tuple.impl.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import profiler.MeasureTools; +import transaction.context.TxnContext; +import transaction.function.Condition; +import transaction.function.DEC; +import transaction.function.INC; +import transaction.impl.ordered.TxnManagerTStream; + +import java.util.ArrayDeque; +import java.util.Map; +import java.util.concurrent.BrokenBarrierException; + +import static common.CONTROL.*; +import static common.constants.OnlineBidingSystemConstants.Constant.NUM_ACCESSES_PER_BUY; +import static profiler.MeasureTools.BEGIN_POST_TIME_MEASURE; +import static profiler.MeasureTools.END_POST_TIME_MEASURE_ACC; +import static profiler.Metrics.NUM_ITEMS; + +public class OBBolt_ts_s extends OBBolt { + private static final Logger LOG= LoggerFactory.getLogger(OBBolt_ts_s.class); + ArrayDeque buyingEvents; + private int alertEvents = 0, toppingEvents = 0; + public OBBolt_ts_s(int fid, SINKCombo sink) { + super(LOG, fid, sink); + } + public OBBolt_ts_s(int fid) { + super(LOG, fid, null); + } + + @Override + public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { + super.initialize(thread_Id, thisTaskId, graph); + transactionManager = new TxnManagerTStream(db.getStorageManager(), this.context.getThisComponentId(), thread_Id, NUM_ITEMS, this.context.getThisComponent().getNumTasks(), config.getString("scheduler", "BL")); + } + public void loadDB(Map conf, TopologyContext context, OutputCollector collector) { +// prepareEvents(); + loadDB(context.getThisTaskId() - context.getThisComponent().getExecutorList().get(0).getExecutorID(), context.getGraph()); + } + + @Override + public void execute(Tuple in) throws InterruptedException, DatabaseException, BrokenBarrierException { + if (in.isMarker()){ + int readSize= buyingEvents.size(); + int num_events=readSize+alertEvents+toppingEvents; + /** + * MeasureTools.BEGIN_TOTAL_TIME_MEASURE(thread_Id); at {@link #execute_ts_normal(Tuple)}}. + */ + MeasureTools.BEGIN_TXN_TIME_MEASURE(thread_Id); + { + transactionManager.start_evaluate(thread_Id,in.getBID(),num_events); + BUYING_REQUEST_CORE(); + } + MeasureTools.END_TXN_TIME_MEASURE(thread_Id); + BUYING_REQUEST_POST(); + MeasureTools.BEGIN_POST_TIME_MEASURE(thread_Id); + BUYING_REQUEST_POST(); + MeasureTools.END_POST_TIME_MEASURE_ACC(thread_Id); + buyingEvents.clear();//all tuples in the EventsHolder are finished. + if (enable_profile) {//all tuples in the holder are finished. + alertEvents = 0; + toppingEvents = 0; + } + }else { + execute_ts_normal(in); + } + } + + @Override + protected void PRE_TXN_PROCESS(long bid, long timestamp) throws DatabaseException, InterruptedException { + MeasureTools.BEGIN_PRE_TXN_TIME_MEASURE(thread_Id); + for (long i = _bid; i < _bid + combo_bid_size; i++) { + TxnContext txnContext = new TxnContext(thread_Id, this.fid, i); + TxnEvent event = (TxnEvent) input_event; + if (enable_latency_measurement) + (event).setTimestamp(timestamp); + if (event instanceof BuyingEvent) { + BUYING_REQUEST_CONSTRUCT((BuyingEvent) event, txnContext); + } else if (event instanceof AlertEvent) { + ALERT_REQUEST_CONSTRUCT((AlertEvent) event, txnContext); + } else { + TOPPING_REQUEST_CONSTRUCT((ToppingEvent) event, txnContext); + } + } + } + private void BUYING_REQUEST_CONSTRUCT(BuyingEvent event, TxnContext txnContext) throws DatabaseException { + //it simply construct the operations and return. + transactionManager.BeginTransaction(txnContext); + for (int i = 0; i < NUM_ACCESSES_PER_BUY; i++) { + //it simply constructs the operations and return. + //condition on itself. + transactionManager.Asy_ModifyRecord(//TODO: addOperation atomicity preserving later. + txnContext, + "goods", + String.valueOf(event.getItemId()[i]), + new DEC(event.getBidQty(i)), + new Condition(event.getBidPrice(i), event.getBidQty(i)), + event.success + ); + } + transactionManager.CommitTransaction(txnContext); + buyingEvents.add(event); + } + protected void ALERT_REQUEST_CONSTRUCT(AlertEvent event, TxnContext txnContext) throws DatabaseException, InterruptedException { + //it simply construct the operations and return. + transactionManager.BeginTransaction(txnContext); + for (int i = 0; i < event.getNum_access(); i++) + transactionManager.Asy_WriteRecord(txnContext, "goods", String.valueOf(event.getItemId()[i]), event.getAsk_price()[i], 1);//asynchronously return. + BEGIN_POST_TIME_MEASURE(thread_Id); + ALERT_REQUEST_POST(event); + END_POST_TIME_MEASURE_ACC(thread_Id); + transactionManager.CommitTransaction(txnContext); + alertEvents++; + } + protected void TOPPING_REQUEST_CONSTRUCT(ToppingEvent event, TxnContext txnContext) throws DatabaseException, InterruptedException { + //it simply construct the operations and return. + transactionManager.BeginTransaction(txnContext); + for (int i = 0; i < event.getNum_access(); i++) + transactionManager.Asy_ModifyRecord(txnContext, "goods", String.valueOf(event.getItemId()[i]), new INC(event.getItemTopUp()[i]), 2);//asynchronously return. + BEGIN_POST_TIME_MEASURE(thread_Id); + TOPPING_REQUEST_POST(event); + END_POST_TIME_MEASURE_ACC(thread_Id); + transactionManager.CommitTransaction(txnContext); + toppingEvents++; + } + private void BUYING_REQUEST_CORE() { + for (BuyingEvent event : buyingEvents) { + BUYING_REQUEST_CORE(event); + } + } + + /** + * Evaluation are pushed down.. + * + * @param event + */ + @Override + protected void BUYING_REQUEST_CORE(BuyingEvent event) { + //measure_end if any item is not able to buy. + event.biding_result = new BidingResult(event, event.success[0] == NUM_ACCESSES_PER_BUY); + } + private void BUYING_REQUEST_POST() throws InterruptedException { + for (BuyingEvent event : buyingEvents) { + BUYING_REQUEST_POST(event); + } + } +} diff --git a/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts.java b/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts.java index 9b462d341..692456ff8 100644 --- a/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts.java +++ b/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts.java @@ -80,7 +80,8 @@ protected void REQUEST_CONSTRUCT(LREvent event, TxnContext txnContext) throws Da @Override public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { super.initialize(thread_Id, thisTaskId, graph); - transactionManager = new TxnManagerTStream(db.getStorageManager(), this.context.getThisComponentId(), thread_Id, NUM_SEGMENTS, this.context.getThisComponent().getNumTasks(), config.getString("scheduler", "BL")); + transactionManager = new TxnManagerTStream(db.getStorageManager(), this.context.getThisComponentId(), thread_Id, + NUM_SEGMENTS, this.context.getThisComponent().getNumTasks(), config.getString("scheduler", "BL")); } @Override diff --git a/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts_s.java b/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts_s.java new file mode 100644 index 000000000..ef8c32f15 --- /dev/null +++ b/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts_s.java @@ -0,0 +1,117 @@ +package common.bolts.transactional.tp; + +import combo.SINKCombo; +import common.param.lr.LREvent; +import components.context.TopologyContext; +import db.DatabaseException; +import execution.ExecutionGraph; +import execution.runtime.collector.OutputCollector; +import execution.runtime.tuple.impl.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import profiler.MeasureTools; +import transaction.context.TxnContext; +import transaction.function.AVG; +import transaction.function.CNT; +import transaction.impl.ordered.TxnManagerTStream; + +import java.util.ArrayDeque; +import java.util.Map; +import java.util.concurrent.BrokenBarrierException; + +import static common.CONTROL.enable_latency_measurement; +import static common.constants.TPConstants.Constant.NUM_SEGMENTS; + +public class TPBolt_ts_s extends TPBolt { + private static final Logger LOG= LoggerFactory.getLogger(TPBolt_ts_s.class); + ArrayDeque LREvents; + public TPBolt_ts_s(int fid,SINKCombo sink){ + super(LOG,fid,sink); + } + public TPBolt_ts_s(int fid){ + super(LOG,fid,null); + } + + @Override + public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { + super.initialize(thread_Id, thisTaskId, graph); + transactionManager=new TxnManagerTStream(db.getStorageManager(), this.context.getThisComponentId(),thread_Id, + NUM_SEGMENTS,this.context.getThisComponent().getNumTasks(),config.getString("scheduler","BF")); + LREvents=new ArrayDeque<>(); + } + public void loadDB(Map conf, TopologyContext context, OutputCollector collector) { + loadDB(context.getThisTaskId() - context.getThisComponent().getExecutorList().get(0).getExecutorID() + , context.getGraph()); + } + + @Override + public void execute(Tuple in) throws InterruptedException, DatabaseException, BrokenBarrierException { + if (in.isMarker()){ + /** + * MeasureTools.BEGIN_TOTAL_TIME_MEASURE(thread_Id); at {@link #execute_ts_normal(Tuple)}}. + */ + int readSize=LREvents.size(); + MeasureTools.BEGIN_TXN_TIME_MEASURE(thread_Id); + { + transactionManager.start_evaluate(thread_Id,in.getBID(),readSize); + REQUEST_REQUEST_CORE(); + } + MeasureTools.END_TXN_TIME_MEASURE(thread_Id); + MeasureTools.BEGIN_POST_TIME_MEASURE(thread_Id); + { + REQUEST_POST(); + } + MeasureTools.END_POST_TIME_MEASURE_ACC(thread_Id); + //all tuples in the holder is finished. + LREvents.clear(); + MeasureTools.END_TOTAL_TIME_MEASURE_TS(thread_Id,readSize); + }else{ + execute_ts_normal(in); + } + } + + @Override + protected void PRE_TXN_PROCESS(long bid, long timestamp) throws DatabaseException, InterruptedException { + MeasureTools.BEGIN_PRE_TXN_TIME_MEASURE(thread_Id); + for(long i=_bid;i<_bid+_combo_bid_size;i++){ + TxnContext txnContext=new TxnContext(thread_Id,this.fid,i); + LREvent event=(LREvent) input_event; + if(enable_latency_measurement){ + (event).setTimestamp(timestamp); + } + REQUEST_CONSTRUCT(event,txnContext); + } + } + protected void REQUEST_CONSTRUCT(LREvent event,TxnContext txnContext) throws DatabaseException{ + transactionManager.BeginTransaction(txnContext); + transactionManager.Asy_ModifyRecord_Read(txnContext + , "segment_speed" + , String.valueOf(event.getPOSReport().getSegment()) + , event.speed_value//holder to be filled up. + , new AVG(event.getPOSReport().getSpeed()) + ); //asynchronously return. + transactionManager.Asy_ModifyRecord_Read(txnContext + , "segment_cnt" + , String.valueOf(event.getPOSReport().getSegment()) + , event.count_value//holder to be filled up. + , new CNT(event.getPOSReport().getVid()) + ); //asynchronously return. + transactionManager.CommitTransaction(txnContext); + LREvents.add(event); + } + protected void REQUEST_REQUEST_CORE() { + for (LREvent event : LREvents) { + TXN_REQUEST_CORE_TS(event); + } + } + + private void TXN_REQUEST_CORE_TS(LREvent event) { + event.count = event.count_value.getRecord().getValue().getInt(); + event.lav = event.speed_value.getRecord().getValue().getDouble(); + } + protected void REQUEST_POST() throws InterruptedException { + for (LREvent event : LREvents) { + REQUEST_POST(event); + } + } +} diff --git a/common/src/main/java/common/Runner.java b/common/src/main/java/common/Runner.java index 49f97ba53..8f00b58d2 100644 --- a/common/src/main/java/common/Runner.java +++ b/common/src/main/java/common/Runner.java @@ -83,6 +83,16 @@ public abstract class Runner implements IRunner { @Parameter(names = {"--idGenType"}, description = "State ids distribution scheme.[uniform, normal]") public String idGenType = "uniform"; + /** + * Dynamic Scheduler + */ + @Parameter(names = {"--isDynamic"}, description = "Dynamic Workload") + public boolean isDynamic = false; + @Parameter(names = {"--schedulerPool"}, description = "Schedulers in the SchedulerPool[OG_DFS,OP_DFS]") + public String schedulerPools = "OG_DFS,OP_DFS"; + @Parameter(names = {"--defaultScheduler"}, description = "Default scheduler") + public String defaultScheduler = "OG_DFS"; + /** * Benchmarking Specific Parameters. */ @@ -205,6 +215,11 @@ public void initializeCfg(HashMap config) { config.put("runtimeInSeconds", runtimeInSeconds); config.put("size_tuple", size_tuple); config.put("verbose", verbose); + + /* Dynamic Workload Configuration*/ + config.put("isDynamic",isDynamic); + config.put("schedulersPool",schedulerPools); + config.put("defaultScheduler",defaultScheduler); System.setProperty("my.log", metric_path); } } diff --git a/state-engine/src/main/java/scheduler/collector/Collector.java b/state-engine/src/main/java/scheduler/collector/Collector.java index 8a61054a8..34859e8da 100644 --- a/state-engine/src/main/java/scheduler/collector/Collector.java +++ b/state-engine/src/main/java/scheduler/collector/Collector.java @@ -7,37 +7,70 @@ public class Collector { Num of TD,LD,PD */ private HashMap Num_of_dependency; + private int B_TD; + private int B_LD; + private int B_PD; + private int B_SUM_D; /* Vertex Degree Distribution */ private double VDD; + private double B_VDD; /* Is Cyclic Dependency */ - private boolean IsCyclicDependency; + private boolean isCyclicDependency; /* Computation Complexity, if high return true */ - private boolean IsComputationComplexity; + private boolean isComputationComplexity; /* Rate of Aborting Vertexes */ private double R_of_A; + private double B_R_of_A; + /** + * Init the collector after each switch + */ + private void InitCollector(){ + } + /** + * Configure the bottom line for triggering scheduler switching + */ + public void setBottomLine(){ + + } /** * Implement the decision tree * Return the decision of the scheduler switching + * "OG_BFS","OG_BFS_A","OP_BFS","OP_BFS_A","OG_NS","OG_NS_A","OP_NS","OP_NS_A", "OG_DFS","OG_DFS_A","OP_DFS","OP_DFS_A" + * 0000 0001 0010 0011 0100 0101 0110 0111 1000 1001 1010 1011 * @return */ - public int getDecision(){ - return 0; + private String decisionTree(int TD,int LD,int PD,double VDD,boolean isCyclicDependency,boolean isComputationComplexity,double R_of_A){ + String schedulers[]={"OG_BFS","OG_BFS_A","OP_BFS","OP_BFS_A","OG_NS","OG_NS_A","OP_NS","OP_NS_A", "OG_DFS","OG_DFS_A","OP_DFS","OP_DFS_A"}; + int flag=0; + if((TD+PD+LD)>B_SUM_D){ + if(VDDB_TD&&PDB_R_of_A){ + flag=flag+1; + } + return schedulers[flag]; } - /** - * Init the collector after each switch - */ - private void InitCollector(){ - + public String getDecisionFromConf(){ + return ""; } + public String getDecisionFromRuntimeInfo(){ + return decisionTree(Num_of_dependency.get(0),Num_of_dependency.get(1),Num_of_dependency.get(2),VDD,isCyclicDependency,isComputationComplexity,R_of_A); + } } diff --git a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java index 1f31e12a4..38c30659d 100644 --- a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java @@ -79,11 +79,14 @@ public TaskPrecedenceGraph(int totalThreads, int delta, int NUM_ITEMS, int app) this.app = app; //create holder. operationChains = new ConcurrentHashMap<>(); - if (app == 0) { + if (app == 0) {//GS operationChains.put("MicroTable", new TableOCs<>(totalThreads)); - } else if (app == 1) { + } else if (app == 1) {//SL operationChains.put("accounts", new TableOCs<>(totalThreads)); operationChains.put("bookEntries", new TableOCs<>(totalThreads)); + } else if(app==2){//TP + operationChains.put("segment_speed",new TableOCs<>(totalThreads)); + operationChains.put("segment_cnt",new TableOCs<>(totalThreads)); } } @@ -115,6 +118,13 @@ public void setOCs(Context context) { operationChains.get("bookEntries").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, beOC); ocs.add(accOC); ocs.add(beOC); + }else if(app==2){ + SchedulingUnit speedOC=context.createTask("segment_speed",_key,0); + SchedulingUnit cntOC=context.createTask("segment_cnt",_key,0); + operationChains.get("segment_speed").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, speedOC); + operationChains.get("segment_cnt").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, cntOC); + ocs.add(speedOC); + ocs.add(cntOC); } } threadToOCs.put(context.thisThreadId, ocs); diff --git a/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java index c3f628fa5..2f7669718 100644 --- a/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java @@ -70,11 +70,14 @@ public TaskPrecedenceGraph(int totalThreads, int delta, int NUM_ITEMS, int app) this.app = app; //create holder. operationChains = new ConcurrentHashMap<>(); - if (app == 0) { + if (app == 0) {//GS operationChains.put("MicroTable", new TableOCs(totalThreads)); - } else if (app == 1) { + } else if (app == 1) {//SL operationChains.put("accounts", new TableOCs(totalThreads)); operationChains.put("bookEntries", new TableOCs(totalThreads)); + } else if(app==2){//TP + operationChains.put("segment_speed",new TableOCs(totalThreads)); + operationChains.put("segment_cnt",new TableOCs(totalThreads)); } } @@ -106,6 +109,13 @@ public void setOCs(Context context) { operationChains.get("bookEntries").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, beOC); ocs.add(accOC); ocs.add(beOC); + } else if(app==2){ + OperationChain speedOC=context.createTask("segment_speed",_key); + OperationChain cntOC=context.createTask("segment_cnt",_key); + operationChains.get("segment_speed").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, speedOC); + operationChains.get("segment_cnt").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, cntOC); + ocs.add(speedOC); + ocs.add(cntOC); } } threadToOCs.put(context.thisThreadId, ocs); diff --git a/state-engine/src/main/java/transaction/TxnManager.java b/state-engine/src/main/java/transaction/TxnManager.java index b62c7fec7..26b851575 100644 --- a/state-engine/src/main/java/transaction/TxnManager.java +++ b/state-engine/src/main/java/transaction/TxnManager.java @@ -15,11 +15,15 @@ import scheduler.impl.op.structured.OPDFSAScheduler; import scheduler.impl.op.structured.OPDFSScheduler; +import java.util.HashMap; +import java.util.List; + /** * Every thread has its own TxnManager. */ public abstract class TxnManager implements ITxnManager { protected static IScheduler scheduler; + protected static HashMap schedulerPool; public static void CreateScheduler(String schedulerType, int threadCount, int numberOfStates, int app) { switch (schedulerType) { @@ -78,9 +82,63 @@ public boolean SwitchScheduler(){ } /** - * Configure the bottom line for triggering scheduled switching in Collector + * Configure the bottom line for triggering scheduler switching in Collector */ public static void setBottomLine(){ } + + /** + * Configure the scheduler pool + */ + public static void initSchedulerPool(String defaultScheduler,String schedulerPool,int threadCount, int numberOfStates, int app){ + TxnManager.schedulerPool=new HashMap<>(); + String[] scheduler =schedulerPool.split(","); + for(int i=0;i(threadCount, numberOfStates, app); + case "OP_NS_A": // Single operation + Non-structured exploration strategy + fine-grained + return new OPNSAScheduler<>(threadCount, numberOfStates, app); + case "OP_BFS": // Single operation + Structured BFS exploration strategy + coarse-grained + return new OPBFSScheduler<>(threadCount, numberOfStates, app); + case "OP_BFS_A": // Single operation + Structured BFS exploration strategy + fine-grained + return new OPBFSAScheduler<>(threadCount, numberOfStates, app); + case "OP_DFS": // Single operation + Structured DFS exploration strategy + coarse-grained + return new OPDFSScheduler<>(threadCount, numberOfStates, app); + case "OP_DFS_A": // Single operation + Structured DFS exploration strategy + fine-grained + return new OPDFSAScheduler<>(threadCount, numberOfStates, app); + case "TStream": // original TStream also uses Non-structured exploration strategy + return new TStreamScheduler(threadCount, numberOfStates, app); + default: + throw new UnsupportedOperationException("unsupported scheduler type: " + schedulerType); + } + } + } diff --git a/stream-engine/src/main/java/execution/ExecutionManager.java b/stream-engine/src/main/java/execution/ExecutionManager.java index bc839cdd7..c96761c59 100644 --- a/stream-engine/src/main/java/execution/ExecutionManager.java +++ b/stream-engine/src/main/java/execution/ExecutionManager.java @@ -9,6 +9,7 @@ import execution.runtime.executorThread; import execution.runtime.spoutThread; import optimization.OptimizationManager; +import org.apache.zookeeper.txn.Txn; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import transaction.TxnManager; @@ -66,9 +67,14 @@ public void distributeTasks(Configuration conf, int numberOfStates = conf.getInt("NUM_ITEMS"); String schedulerType = conf.getString("scheduler"); int app = conf.getInt("app"); - TxnManager.CreateScheduler(schedulerType, totalThread, numberOfStates, app); - //TODO: Configure the bottom line for triggering scheduler switching in Collector - TxnManager.setBottomLine(); + String schedulers=conf.getString("schedulersPool"); + if(conf.getBoolean("isDynamic")){ + //TODO: Configure the bottom line for triggering scheduler switching in Collector + TxnManager.initSchedulerPool(conf.getString("defaultScheduler"),schedulers, totalThread,numberOfStates,app); + TxnManager.setBottomLine(); + }else{ + TxnManager.CreateScheduler(schedulerType, totalThread, numberOfStates, app); + } } } executorThread thread = null; From 6346d6a40dfd914217662c7529b4429349f84d2f Mon Sep 17 00:00:00 2001 From: curryzjj Date: Mon, 14 Mar 2022 16:04:52 +0800 Subject: [PATCH 03/15] Finish dynamic scheduler --- common/src/main/java/common/Runner.java | 22 +++-- .../java/scheduler/collector/Collector.java | 88 ++++++++++++++++--- .../java/scheduler/impl/og/OGScheduler.java | 5 +- .../impl/og/nonstructured/OGNSAScheduler.java | 4 + .../impl/og/nonstructured/OGNSScheduler.java | 4 + .../impl/og/structured/OGBFSAScheduler.java | 4 + .../impl/og/structured/OGBFSScheduler.java | 4 + .../impl/og/structured/OGDFSAScheduler.java | 4 + .../impl/og/structured/OGDFSScheduler.java | 4 + .../java/scheduler/impl/op/OPScheduler.java | 10 ++- .../struct/og/TaskPrecedenceGraph.java | 1 + .../struct/og/nonstructured/NSAOperation.java | 5 ++ .../struct/og/nonstructured/NSOperation.java | 6 ++ .../og/structured/bfs/BFSOperation.java | 5 ++ .../og/structured/dfs/DFSOperation.java | 5 ++ .../java/scheduler/struct/op/Operation.java | 5 ++ .../struct/op/TaskPrecedenceGraph.java | 1 + .../src/main/java/transaction/TxnManager.java | 30 +++++-- .../impl/TxnManagerDedicatedAsy.java | 8 +- .../impl/ordered/TxnManagerTStream.java | 15 +++- .../src/main/java/utils/SOURCE_CONTROL.java | 13 +++ .../main/java/execution/ExecutionManager.java | 7 +- 22 files changed, 214 insertions(+), 36 deletions(-) diff --git a/common/src/main/java/common/Runner.java b/common/src/main/java/common/Runner.java index 8f00b58d2..5fe447d01 100644 --- a/common/src/main/java/common/Runner.java +++ b/common/src/main/java/common/Runner.java @@ -57,7 +57,7 @@ public abstract class Runner implements IRunner { * TStream Specific Parameters. */ @Parameter(names = {"--tthread"}, description = "total execution threads") - public int tthread = 1;// default total execution threads + public int tthread = 2;// default total execution threads @Parameter(names = {"--CCOption"}, description = "Selecting different concurrency control options.") public int CCOption = CCOption_MorphStream; // public int CCOption = CCOption_SStore; @@ -65,14 +65,14 @@ public abstract class Runner implements IRunner { @Parameter(names = {"--partition"}, description = "Partitioning database. It must be enabled for S-Store scheme and it is optional for TStream scheme.") public boolean enable_partition = false; @Parameter(names = {"--scheduler"}, description = "Scheduler for TStream.") - public String scheduler = "OG_BFS"; + // public String scheduler = "OG_DFS" // public String scheduler = "OG_BFS_A"; // public String scheduler = "OG_DFS"; // public String scheduler = "OG_DFS_A"; // public String scheduler = "OG_NS"; // public String scheduler = "OG_NS_A"; // public String scheduler = "OP_NS"; -// public String scheduler = "OP_NS_A"; + public String scheduler = "OP_NS_A"; // public String scheduler = "OP_BFS"; // public String scheduler = "OP_BFS_A"; // public String scheduler = "OP_DFS"; @@ -86,12 +86,18 @@ public abstract class Runner implements IRunner { /** * Dynamic Scheduler */ + @Parameter(names = {"--isRuntime"}, description = "Collect runtime information") + public boolean isRuntime = false; @Parameter(names = {"--isDynamic"}, description = "Dynamic Workload") - public boolean isDynamic = false; + public boolean isDynamic = true; @Parameter(names = {"--schedulerPool"}, description = "Schedulers in the SchedulerPool[OG_DFS,OP_DFS]") - public String schedulerPools = "OG_DFS,OP_DFS"; + public String schedulerPools = "OP_NS_A,OP_BFS_A,OP_BFS,OG_BFS"; @Parameter(names = {"--defaultScheduler"}, description = "Default scheduler") public String defaultScheduler = "OG_DFS"; + @Parameter(names = {"--bottomLine"}, description = "BottomLine for(TD,LD,PD,SUM,VDD,R_of_A)") + public String bottomLine = "2,3,5,10,0.6,0.7"; + @Parameter(names = {"--WorkloadConfig"}, description = "WorkloadConfigs(TD,LD,PD,VDD,R_of_A,isCD,isCC,markId)") + public String WorkloadConfig = "1,3,4,0.5,0.6,1,1,19999;1,6,4,0.5,0.7,1,1,39999;3,6,3,0.5,0.7,1,1,59999;3,6,3,0.5,0.8,0,0,100000"; /** * Benchmarking Specific Parameters. @@ -219,7 +225,11 @@ public void initializeCfg(HashMap config) { /* Dynamic Workload Configuration*/ config.put("isDynamic",isDynamic); config.put("schedulersPool",schedulerPools); - config.put("defaultScheduler",defaultScheduler); + config.put("defaultScheduler",scheduler); + config.put("isRuntime",isRuntime); + config.put("bottomLine",bottomLine); + config.put("WorkloadConfig",WorkloadConfig); + System.setProperty("my.log", metric_path); } } diff --git a/state-engine/src/main/java/scheduler/collector/Collector.java b/state-engine/src/main/java/scheduler/collector/Collector.java index 34859e8da..b09bf18fc 100644 --- a/state-engine/src/main/java/scheduler/collector/Collector.java +++ b/state-engine/src/main/java/scheduler/collector/Collector.java @@ -1,6 +1,8 @@ package scheduler.collector; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; public class Collector { /* @@ -29,27 +31,51 @@ public class Collector { */ private double R_of_A; private double B_R_of_A; + private boolean isRuntime=true; + private final List workloadConfig=new ArrayList<>(); + private final HashMap currentWorkload=new HashMap<>(); /** - * Init the collector after each switch + * Init the collector */ - private void InitCollector(){ + public void InitCollector(int threadCount){ + for (int i=0;iB_SUM_D){ if(VDDB_TD&&PDB_PD){ flag=flag+2; } - if(!isComputationComplexity&&R_of_A>B_R_of_A){ + if(isComputationComplexity||R_of_A=currentId){ + currentWorkload.put(threadId,currentWorkload.get(threadId)+1); + return true; + }else + return false; + } + } } diff --git a/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java b/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java index e40ddf384..f4fdad90c 100644 --- a/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java @@ -326,7 +326,10 @@ public void TxnSubmitBegin(Context context) { @Override public void AddContext(int threadId, Context context) { tpg.threadToContextMap.put(threadId, context); - tpg.setOCs(context); + /*Thread to OCs does not need reconfigure*/ + if (!tpg.isThreadTOCsReady()){ + tpg.setOCs(context); + } } protected boolean isConflicted(Context context, SchedulingUnit operationChain, ExecutionUnit operation) { diff --git a/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSAScheduler.java b/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSAScheduler.java index 0bf0d94f4..6ca7876e1 100644 --- a/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSAScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSAScheduler.java @@ -79,6 +79,10 @@ private NSAOperation constructOp(List operationGraph, Request requ set_op = new NSAOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); break; + case READ_WRITE_READ: + set_op=new NSAOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, + request.d_record, request.record_ref, request.function); + break; default: throw new RuntimeException("Unexpected operation"); } diff --git a/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSScheduler.java b/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSScheduler.java index 5dbd058b8..404034df5 100644 --- a/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSScheduler.java @@ -110,6 +110,10 @@ private void constructOp(List operationGraph, Request request, OGNS set_op = new NSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); break; + case READ_WRITE_READ: + set_op = new NSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, + request.d_record, request.record_ref, request.function); + break; default: throw new RuntimeException("Unexpected operation"); } diff --git a/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSAScheduler.java b/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSAScheduler.java index f7dca39f5..b287eecdb 100644 --- a/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSAScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSAScheduler.java @@ -134,6 +134,10 @@ private void constructOp(List operationGraph, Request request) { set_op = new BFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); break; + case READ_WRITE_READ: + set_op = new BFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, + request.d_record, request.record_ref, request.function); + break; default: throw new UnsupportedOperationException(); } diff --git a/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSScheduler.java b/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSScheduler.java index 9a83f742f..f604a2780 100644 --- a/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSScheduler.java @@ -66,6 +66,10 @@ private void constructOp(List operationGraph, Request request) { set_op = new BFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); break; + case READ_WRITE_READ: + set_op = new BFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, + request.d_record, request.record_ref, request.function); + break; default: throw new UnsupportedOperationException(); } diff --git a/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSAScheduler.java b/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSAScheduler.java index 437961b87..fbb6351bc 100644 --- a/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSAScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSAScheduler.java @@ -138,6 +138,10 @@ private void constructOp(List operationGraph, Request request) { set_op = new DFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); break; + case READ_WRITE_READ: + set_op = new DFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, + request.d_record, request.record_ref, request.function); + break; default: throw new UnsupportedOperationException(); } diff --git a/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSScheduler.java b/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSScheduler.java index 032a6b06b..ecc356cf0 100644 --- a/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSScheduler.java @@ -66,6 +66,10 @@ private void constructOp(List operationGraph, Request request) { set_op = new DFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); break; + case READ_WRITE_READ: + set_op = new DFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, + request.d_record, request.record_ref, request.function); + break; default: throw new UnsupportedOperationException(); } diff --git a/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java b/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java index 75fd13a9c..02d6a1342 100644 --- a/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java @@ -197,9 +197,13 @@ protected void GrepSum_Fun(AbstractOperation operation, long previous_mark_ID, b @Override public void AddContext(int threadId, Context context) { tpg.threadToContextMap.put(threadId, context); - tpg.setOCs(context); + /*Thread to OCs does not need reconfigure*/ + if (!tpg.isThreadTOCsReady()){ + tpg.setOCs(context); + } } + /** * Submit requests to target thread --> data shuffling is involved. * @@ -242,6 +246,10 @@ public void TxnSubmitFinished(Context context) { set_op = new Operation(request.src_key, getTargetContext(request.src_key), request.table_name, request.txn_context, bid, request.accessType, request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); break; + case READ_WRITE_READ: + set_op = new Operation(request.src_key, getTargetContext(request.src_key), request.table_name, request.txn_context, bid, request.accessType, + request.d_record, request.record_ref, request.function); + break; default: throw new UnsupportedOperationException(); } diff --git a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java index 38c30659d..360420dfa 100644 --- a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java @@ -133,6 +133,7 @@ public void setOCs(Context context) { public TableOCs getTableOCs(String table_name) { return operationChains.get(table_name); } + public boolean isThreadTOCsReady(){return threadToOCs.size()==totalThreads; } public ConcurrentHashMap> getOperationChains() { return operationChains; diff --git a/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSAOperation.java b/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSAOperation.java index a01ca41f0..9f561c61e 100644 --- a/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSAOperation.java +++ b/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSAOperation.java @@ -55,6 +55,11 @@ public NSAOperation(String pKey, Context c SchemaRecordRef record_ref) { this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, null, null, null, null); } + public NSAOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, + CommonMetaTypes.AccessType accessType, TableRecord d_record, + SchemaRecordRef record_ref,Function function) { + this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, function, null, null, null); + } public NSAOperation( String pKey, Context context, String table_name, TxnContext txn_context, long bid, diff --git a/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSOperation.java b/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSOperation.java index df46314b0..fd8662e5c 100644 --- a/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSOperation.java +++ b/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSOperation.java @@ -46,6 +46,12 @@ public NSOperation(String pKey, Context co SchemaRecordRef record_ref) { this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, null, null, null, null); } + public NSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, + CommonMetaTypes.AccessType accessType, TableRecord d_record, + SchemaRecordRef record_ref,Function function) { + this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, function, null, null, null); + } + public NSOperation( String pKey, Context context, String table_name, TxnContext txn_context, long bid, diff --git a/state-engine/src/main/java/scheduler/struct/og/structured/bfs/BFSOperation.java b/state-engine/src/main/java/scheduler/struct/og/structured/bfs/BFSOperation.java index 7a37a9f1c..5ead2d638 100644 --- a/state-engine/src/main/java/scheduler/struct/og/structured/bfs/BFSOperation.java +++ b/state-engine/src/main/java/scheduler/struct/og/structured/bfs/BFSOperation.java @@ -46,6 +46,11 @@ public BFSOperation(String pKey, Context co SchemaRecordRef record_ref) { this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, null, null, null, null); } + public BFSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, + CommonMetaTypes.AccessType accessType, TableRecord d_record, + SchemaRecordRef record_ref,Function function) { + this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, function, null, null, null); + } public BFSOperation( String pKey, Context context, String table_name, TxnContext txn_context, long bid, diff --git a/state-engine/src/main/java/scheduler/struct/og/structured/dfs/DFSOperation.java b/state-engine/src/main/java/scheduler/struct/og/structured/dfs/DFSOperation.java index 6cd748bcf..230f7db7d 100644 --- a/state-engine/src/main/java/scheduler/struct/og/structured/dfs/DFSOperation.java +++ b/state-engine/src/main/java/scheduler/struct/og/structured/dfs/DFSOperation.java @@ -56,6 +56,11 @@ public DFSOperation(String pKey, Context co SchemaRecordRef record_ref) { this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, null, null, null, null); } + public DFSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, + CommonMetaTypes.AccessType accessType, TableRecord d_record, + SchemaRecordRef record_ref,Function function) { + this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, function, null, null, null); + } public DFSOperation( String pKey, Context context, String table_name, TxnContext txn_context, long bid, diff --git a/state-engine/src/main/java/scheduler/struct/op/Operation.java b/state-engine/src/main/java/scheduler/struct/op/Operation.java index 55c93dfc4..d0ab979eb 100644 --- a/state-engine/src/main/java/scheduler/struct/op/Operation.java +++ b/state-engine/src/main/java/scheduler/struct/op/Operation.java @@ -87,6 +87,11 @@ public Operation(String pKey, Context conte SchemaRecordRef record_ref) { this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, null, null, null, null); } + public Operation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, + CommonMetaTypes.AccessType accessType, TableRecord d_record, + SchemaRecordRef record_ref,Function function) { + this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, function, null, null, null); + } public Operation( String pKey, Context context, String table_name, TxnContext txn_context, long bid, diff --git a/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java index 2f7669718..3516b99a2 100644 --- a/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java @@ -124,6 +124,7 @@ public void setOCs(Context context) { public TableOCs getTableOCs(String table_name) { return operationChains.get(table_name); } + public boolean isThreadTOCsReady(){return threadToOCs.size()==totalThreads; } public ConcurrentHashMap getOperationChains() { return operationChains; diff --git a/state-engine/src/main/java/transaction/TxnManager.java b/state-engine/src/main/java/transaction/TxnManager.java index 26b851575..f30c8ea4c 100644 --- a/state-engine/src/main/java/transaction/TxnManager.java +++ b/state-engine/src/main/java/transaction/TxnManager.java @@ -1,5 +1,8 @@ package transaction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scheduler.collector.Collector; import scheduler.impl.IScheduler; import scheduler.impl.og.structured.OGBFSScheduler; import scheduler.impl.og.structured.OGBFSAScheduler; @@ -16,15 +19,20 @@ import scheduler.impl.op.structured.OPDFSScheduler; import java.util.HashMap; -import java.util.List; + +import static common.CONTROL.enable_log; /** * Every thread has its own TxnManager. */ public abstract class TxnManager implements ITxnManager { + private static final Logger log = LoggerFactory.getLogger(TxnManager.class); protected static IScheduler scheduler; + protected static HashMap schedulerPool; + protected static Collector collector=new Collector(); + public static void CreateScheduler(String schedulerType, int threadCount, int numberOfStates, int app) { switch (schedulerType) { case "OG_BFS": // Group of operation + Structured BFS exploration strategy + coarse-grained @@ -74,18 +82,22 @@ public static void CreateScheduler(String schedulerType, int threadCount, int nu /** * Switch scheduler every punctuation * When the workload changes and the scheduler is no longer applicable - * @return */ - public boolean SwitchScheduler(){ - //TODO: implement the scheduler switching - return true; + public void SwitchScheduler(String schedulerType){ + scheduler=schedulerPool.get(schedulerType); } /** * Configure the bottom line for triggering scheduler switching in Collector */ - public static void setBottomLine(){ - + public static void setBottomLine(String bottomLine){ + collector.setBottomLine(bottomLine); + } + /** + * Configure the bottom line for triggering scheduler switching in Collector + */ + public static void setWorkloadConfig(String config){ + collector.setWorkloadConfig(config); } /** @@ -95,9 +107,11 @@ public static void initSchedulerPool(String defaultScheduler,String schedulerPoo TxnManager.schedulerPool=new HashMap<>(); String[] scheduler =schedulerPool.split(","); for(int i=0;i iteration; @@ -30,6 +33,7 @@ public void config(int number_threads) { startBarrier = new CyclicBarrier(number_threads); endBarrier = new CyclicBarrier(number_threads); finalEndBarrier = new CyclicBarrier(number_threads); + switchSchedulerBarrier=new CyclicBarrier(number_threads); dLevelEndBarrier = new Phaser(number_threads); iteration = new HashMap<>(); for (int i = 0; i < number_threads; i++) { @@ -82,6 +86,15 @@ public void waitForOtherThreadsAbort() { ex.printStackTrace(); } } + public void waitForSchedulerSwitch(int threadId){ + try{ + switchSchedulerBarrier.await(); + } catch (BrokenBarrierException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } public void oneThreadCompleted() { dLevelEndBarrier.arriveAndDeregister(); diff --git a/stream-engine/src/main/java/execution/ExecutionManager.java b/stream-engine/src/main/java/execution/ExecutionManager.java index c96761c59..c02b928ff 100644 --- a/stream-engine/src/main/java/execution/ExecutionManager.java +++ b/stream-engine/src/main/java/execution/ExecutionManager.java @@ -69,9 +69,12 @@ public void distributeTasks(Configuration conf, int app = conf.getInt("app"); String schedulers=conf.getString("schedulersPool"); if(conf.getBoolean("isDynamic")){ - //TODO: Configure the bottom line for triggering scheduler switching in Collector TxnManager.initSchedulerPool(conf.getString("defaultScheduler"),schedulers, totalThread,numberOfStates,app); - TxnManager.setBottomLine(); + //Configure the bottom line for triggering scheduler switching in Collector(include the isRuntime and when to switch) + TxnManager.setBottomLine(conf.getString("bottomLine")); + if(!conf.getBoolean("isRuntime")){ + TxnManager.setWorkloadConfig(conf.getString("WorkloadConfig")); + } }else{ TxnManager.CreateScheduler(schedulerType, totalThread, numberOfStates, app); } From 91db916c57cec3171f5c7ff4bcfa2b835fe3acd7 Mon Sep 17 00:00:00 2001 From: curryzjj Date: Mon, 14 Mar 2022 16:14:44 +0800 Subject: [PATCH 04/15] Only the thread-0 --- state-engine/src/main/java/transaction/TxnManager.java | 2 ++ .../main/java/transaction/impl/ordered/TxnManagerTStream.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/state-engine/src/main/java/transaction/TxnManager.java b/state-engine/src/main/java/transaction/TxnManager.java index f30c8ea4c..2b3030bd5 100644 --- a/state-engine/src/main/java/transaction/TxnManager.java +++ b/state-engine/src/main/java/transaction/TxnManager.java @@ -32,6 +32,7 @@ public abstract class TxnManager implements ITxnManager { protected static HashMap schedulerPool; protected static Collector collector=new Collector(); + protected static String currentSchedulerType; public static void CreateScheduler(String schedulerType, int threadCount, int numberOfStates, int app) { switch (schedulerType) { @@ -85,6 +86,7 @@ public static void CreateScheduler(String schedulerType, int threadCount, int nu */ public void SwitchScheduler(String schedulerType){ scheduler=schedulerPool.get(schedulerType); + currentSchedulerType=schedulerType; } /** diff --git a/state-engine/src/main/java/transaction/impl/ordered/TxnManagerTStream.java b/state-engine/src/main/java/transaction/impl/ordered/TxnManagerTStream.java index 8d4ee056d..186ee918e 100644 --- a/state-engine/src/main/java/transaction/impl/ordered/TxnManagerTStream.java +++ b/state-engine/src/main/java/transaction/impl/ordered/TxnManagerTStream.java @@ -64,13 +64,13 @@ public void start_evaluate(int thread_Id, long mark_ID, int num_events) throws I MeasureTools.SCHEDULE_TIME_RECORD(thread_Id, num_events); //Sync to switch scheduler(more overhead) decide by the mark_ID or runtime information if(collector.timeToSwitch(mark_ID,thread_Id)){ - String schedulerType=collector.getDecision(thread_Id); if(thread_Id==0){ + String schedulerType=collector.getDecision(thread_Id); this.SwitchScheduler(schedulerType); if(enable_log) log.info("Current Scheduler is "+schedulerType+" MarkId "+mark_ID); } SOURCE_CONTROL.getInstance().waitForSchedulerSwitch(thread_Id); - this.setSchedulerContext(thread_Id, (int) thread_count_,schedulerType); + this.setSchedulerContext(thread_Id, (int) thread_count_,currentSchedulerType); } } } From e03880fdfdd436a0441032d8cc392165d527aa72 Mon Sep 17 00:00:00 2001 From: curryzjj Date: Mon, 14 Mar 2022 21:18:08 +0800 Subject: [PATCH 05/15] Fix concurrent access bug --- common/src/main/java/common/Runner.java | 2 +- .../main/java/scheduler/collector/Collector.java | 15 ++++++++++----- .../scheduler/struct/og/TaskPrecedenceGraph.java | 2 +- .../scheduler/struct/op/TaskPrecedenceGraph.java | 2 +- .../src/main/java/execution/ExecutionManager.java | 2 +- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/common/src/main/java/common/Runner.java b/common/src/main/java/common/Runner.java index 5fe447d01..ba987c3c3 100644 --- a/common/src/main/java/common/Runner.java +++ b/common/src/main/java/common/Runner.java @@ -121,7 +121,7 @@ public abstract class Runner implements IRunner { * generator parameters */ @Parameter(names = {"--checkpoint_interval"}, description = "checkpoint interval (#tuples)") - public int checkpoint_interval = 20000;//checkpoint per thread. + public int checkpoint_interval = 10000;//checkpoint per thread. @Parameter(names = {"--generator"}, description = "Generator for TStream.") public String generator = "TPGGenerator"; // public String generator = "OCGenerator"; diff --git a/state-engine/src/main/java/scheduler/collector/Collector.java b/state-engine/src/main/java/scheduler/collector/Collector.java index b09bf18fc..afaf3df63 100644 --- a/state-engine/src/main/java/scheduler/collector/Collector.java +++ b/state-engine/src/main/java/scheduler/collector/Collector.java @@ -3,7 +3,10 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; - +/** + * Gather information for decision trees and determine when to switch and which scheduler to switch to + * Created by curry on 3/10/2022. + */ public class Collector { /* Num of TD,LD,PD @@ -32,12 +35,14 @@ public class Collector { private double R_of_A; private double B_R_of_A; private boolean isRuntime=true; + private int threadCount; private final List workloadConfig=new ArrayList<>(); private final HashMap currentWorkload=new HashMap<>(); /** * Init the collector */ public void InitCollector(int threadCount){ + this.threadCount=threadCount; for (int i=0;i=currentId){ + if (currentId<=markId+threadCount-threadId-1){ currentWorkload.put(threadId,currentWorkload.get(threadId)+1); return true; }else diff --git a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java index 360420dfa..8d4219105 100644 --- a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java @@ -73,7 +73,7 @@ public TaskPrecedenceGraph(int totalThreads, int delta, int NUM_ITEMS, int app) this.totalThreads = totalThreads; this.delta = delta; this.NUM_ITEMS = NUM_ITEMS; - threadToContextMap = new HashMap<>(); + threadToContextMap = new ConcurrentHashMap<>(); threadToOCs = new ConcurrentHashMap<>(); //shared data structure. this.app = app; diff --git a/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java index 3516b99a2..c69cd3f1a 100644 --- a/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java @@ -65,7 +65,7 @@ public TaskPrecedenceGraph(int totalThreads, int delta, int NUM_ITEMS, int app) this.delta = delta; this.NUM_ITEMS = NUM_ITEMS; // all parameters in this class should be thread safe. - threadToContextMap = new HashMap<>(); + threadToContextMap = new ConcurrentHashMap<>(); threadToOCs = new ConcurrentHashMap<>(); this.app = app; //create holder. diff --git a/stream-engine/src/main/java/execution/ExecutionManager.java b/stream-engine/src/main/java/execution/ExecutionManager.java index c02b928ff..9a5efa48f 100644 --- a/stream-engine/src/main/java/execution/ExecutionManager.java +++ b/stream-engine/src/main/java/execution/ExecutionManager.java @@ -67,8 +67,8 @@ public void distributeTasks(Configuration conf, int numberOfStates = conf.getInt("NUM_ITEMS"); String schedulerType = conf.getString("scheduler"); int app = conf.getInt("app"); - String schedulers=conf.getString("schedulersPool"); if(conf.getBoolean("isDynamic")){ + String schedulers=conf.getString("schedulersPool"); TxnManager.initSchedulerPool(conf.getString("defaultScheduler"),schedulers, totalThread,numberOfStates,app); //Configure the bottom line for triggering scheduler switching in Collector(include the isRuntime and when to switch) TxnManager.setBottomLine(conf.getString("bottomLine")); From 4bab13f0c38b7ede6246cd4f9994a33b8e1ba9b3 Mon Sep 17 00:00:00 2001 From: curryzjj Date: Tue, 15 Mar 2022 09:03:34 +0800 Subject: [PATCH 06/15] Add a configuration --- common/src/main/java/common/Runner.java | 6 +++--- state-engine/src/main/java/transaction/TxnManager.java | 2 ++ .../java/transaction/impl/ordered/TxnManagerTStream.java | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/common/src/main/java/common/Runner.java b/common/src/main/java/common/Runner.java index ba987c3c3..7f35ffd57 100644 --- a/common/src/main/java/common/Runner.java +++ b/common/src/main/java/common/Runner.java @@ -57,7 +57,7 @@ public abstract class Runner implements IRunner { * TStream Specific Parameters. */ @Parameter(names = {"--tthread"}, description = "total execution threads") - public int tthread = 2;// default total execution threads + public int tthread = 4;// default total execution threads @Parameter(names = {"--CCOption"}, description = "Selecting different concurrency control options.") public int CCOption = CCOption_MorphStream; // public int CCOption = CCOption_SStore; @@ -89,7 +89,7 @@ public abstract class Runner implements IRunner { @Parameter(names = {"--isRuntime"}, description = "Collect runtime information") public boolean isRuntime = false; @Parameter(names = {"--isDynamic"}, description = "Dynamic Workload") - public boolean isDynamic = true; + public boolean isDynamic = false; @Parameter(names = {"--schedulerPool"}, description = "Schedulers in the SchedulerPool[OG_DFS,OP_DFS]") public String schedulerPools = "OP_NS_A,OP_BFS_A,OP_BFS,OG_BFS"; @Parameter(names = {"--defaultScheduler"}, description = "Default scheduler") @@ -97,7 +97,7 @@ public abstract class Runner implements IRunner { @Parameter(names = {"--bottomLine"}, description = "BottomLine for(TD,LD,PD,SUM,VDD,R_of_A)") public String bottomLine = "2,3,5,10,0.6,0.7"; @Parameter(names = {"--WorkloadConfig"}, description = "WorkloadConfigs(TD,LD,PD,VDD,R_of_A,isCD,isCC,markId)") - public String WorkloadConfig = "1,3,4,0.5,0.6,1,1,19999;1,6,4,0.5,0.7,1,1,39999;3,6,3,0.5,0.7,1,1,59999;3,6,3,0.5,0.8,0,0,100000"; + public String WorkloadConfig = "1,3,4,0.5,0.6,1,1,39999;1,6,4,0.5,0.7,1,1,79999;3,6,3,0.5,0.7,1,1,100000"; /** * Benchmarking Specific Parameters. diff --git a/state-engine/src/main/java/transaction/TxnManager.java b/state-engine/src/main/java/transaction/TxnManager.java index 2b3030bd5..14bd4bb6e 100644 --- a/state-engine/src/main/java/transaction/TxnManager.java +++ b/state-engine/src/main/java/transaction/TxnManager.java @@ -30,6 +30,7 @@ public abstract class TxnManager implements ITxnManager { protected static IScheduler scheduler; protected static HashMap schedulerPool; + protected static boolean enableDynamic=false; protected static Collector collector=new Collector(); protected static String currentSchedulerType; @@ -113,6 +114,7 @@ public static void initSchedulerPool(String defaultScheduler,String schedulerPoo } collector.InitCollector(threadCount); TxnManager.scheduler=TxnManager.schedulerPool.get(defaultScheduler); + enableDynamic=true; if(enable_log) log.info("Current Scheduler is "+defaultScheduler); } diff --git a/state-engine/src/main/java/transaction/impl/ordered/TxnManagerTStream.java b/state-engine/src/main/java/transaction/impl/ordered/TxnManagerTStream.java index 186ee918e..499e42fac 100644 --- a/state-engine/src/main/java/transaction/impl/ordered/TxnManagerTStream.java +++ b/state-engine/src/main/java/transaction/impl/ordered/TxnManagerTStream.java @@ -63,7 +63,7 @@ public void start_evaluate(int thread_Id, long mark_ID, int num_events) throws I MeasureTools.END_SCHEDULE_EXPLORE_TIME_MEASURE(thread_Id); MeasureTools.SCHEDULE_TIME_RECORD(thread_Id, num_events); //Sync to switch scheduler(more overhead) decide by the mark_ID or runtime information - if(collector.timeToSwitch(mark_ID,thread_Id)){ + if(enableDynamic&&collector.timeToSwitch(mark_ID,thread_Id)){ if(thread_Id==0){ String schedulerType=collector.getDecision(thread_Id); this.SwitchScheduler(schedulerType); From 19f92201f5de78dd418184119c04468bea393a7b Mon Sep 17 00:00:00 2001 From: Mao Yancan Date: Sun, 13 Mar 2022 11:29:07 +0800 Subject: [PATCH 07/15] System sensitivity study (#86) * unified the namespace, make it be consistent with terms in paper. todo: update scripts. * update * add scripts for system sensitivity study * code clean * updated countdown to atomic integer to fix the bug for DFS * refactor data structures, operations are extending the same operation now. * update\ * unified the namespace, make it be consistent with terms in paper. todo: update scripts. * update * add scripts for system sensitivity study * code clean * updated countdown to atomic integer to fix the bug for DFS * refactor data structures, operations are extending the same operation now. * update\ * code clean * update experiment scripts * unified the namespace, make it be consistent with terms in paper. todo: update scripts. * update * add scripts for system sensitivity study * updated countdown to atomic integer to fix the bug for DFS * refactor data structures, operations are extending the same operation now. * update\ * code clean * unified the namespace, make it be consistent with terms in paper. todo: update scripts. * update * add scripts for system sensitivity study * updated countdown to atomic integer to fix the bug for DFS * refactor data structures, operations are extending the same operation now. * update\ * update experiment scripts * solve conflict Co-authored-by: Aqif Hamid --- .../model/model_exploration_strategy_batch.py | 209 ++++++++++++ .../draw/model/model_granularity_access.py | 27 +- scripts/draw/model/model_granularity_batch.py | 43 +-- .../draw/model/model_granularity_cyclic.py | 39 +-- .../draw/model/model_granularity_writeonly.py | 27 +- scripts/model_runner.sh | 317 +++++++++--------- scripts/runner.sh | 197 +++++++++++ .../context/og/AbstractOGNSContext.java | 23 +- .../scheduler/context/og/OGBFSContext.java | 20 -- .../scheduler/context/og/OGDFSAContext.java | 12 - .../scheduler/context/og/OGDFSContext.java | 20 -- .../scheduler/context/og/OGNSAContext.java | 14 +- .../scheduler/context/og/OGNSContext.java | 16 +- .../{OGBFSAContext.java => OGSAContext.java} | 4 +- .../java/scheduler/context/og/OGSContext.java | 32 +- .../context/og/OGSchedulerContext.java | 9 +- .../java/scheduler/context/op/OPSContext.java | 2 +- .../java/scheduler/impl/og/OGScheduler.java | 82 +++-- .../nonstructured/AbstractOGNSScheduler.java | 27 +- .../nonstructured/ExecutableTaskListener.java | 15 +- .../impl/og/nonstructured/OGNSAScheduler.java | 78 +---- .../impl/og/nonstructured/OGNSScheduler.java | 59 +--- .../og/structured/AbstractOGBFSScheduler.java | 15 +- .../og/structured/AbstractOGDFSScheduler.java | 19 +- .../impl/og/structured/OGBFSAScheduler.java | 95 ++---- .../impl/og/structured/OGBFSScheduler.java | 66 +--- .../impl/og/structured/OGDFSAScheduler.java | 104 ++---- .../impl/og/structured/OGDFSScheduler.java | 62 +--- .../impl/og/structured/OGSScheduler.java | 25 +- .../java/scheduler/impl/op/OPScheduler.java | 2 +- .../impl/op/structured/OPDFSScheduler.java | 2 +- .../impl/op/structured/OPSScheduler.java | 6 +- .../scheduler/signal/oc/OnExecutedSignal.java | 8 +- .../oc/OnHeaderStartAbortHandlingSignal.java | 11 +- .../signal/oc/OnNeedAbortHandlingSignal.java | 11 +- .../signal/oc/OnParentExecutedSignal.java | 7 +- .../signal/oc/OnRollbackAndRedoSignal.java | 7 +- .../scheduler/signal/oc/OnRootSignal.java | 7 +- .../signal/oc/OperationChainSignal.java | 11 +- .../og/OperationChainStateListener.java | 9 +- .../og/PartitionStateManager.java | 31 +- .../og/PartitionStateManagerWithAbort.java | 68 ++-- .../struct/{op => }/AbstractOperation.java | 2 +- .../struct/og/AbstractOperation.java | 111 ------ .../java/scheduler/struct/og/Operation.java | 138 ++++++++ .../scheduler/struct/og/OperationChain.java | 133 ++++++-- .../struct/og/TaskPrecedenceGraph.java | 167 +++++---- .../AbstractNSOperationChain.java | 34 -- .../struct/og/nonstructured/NSAOperation.java | 108 ------ .../og/nonstructured/NSAOperationChain.java | 37 -- .../struct/og/nonstructured/NSOperation.java | 63 ---- .../og/nonstructured/NSOperationChain.java | 18 - .../struct/og/structured/SOperationChain.java | 56 ---- .../og/structured/bfs/BFSOperation.java | 62 ---- .../og/structured/bfs/BFSOperationChain.java | 14 - .../og/structured/dfs/DFSOperation.java | 79 ----- .../og/structured/dfs/DFSOperationChain.java | 29 -- .../java/scheduler/struct/op/Operation.java | 130 +++---- .../impl/TxnManagerDedicatedAsy.java | 12 +- 59 files changed, 1333 insertions(+), 1698 deletions(-) create mode 100644 scripts/draw/model/model_exploration_strategy_batch.py create mode 100644 scripts/runner.sh delete mode 100644 state-engine/src/main/java/scheduler/context/og/OGBFSContext.java delete mode 100644 state-engine/src/main/java/scheduler/context/og/OGDFSAContext.java delete mode 100644 state-engine/src/main/java/scheduler/context/og/OGDFSContext.java rename state-engine/src/main/java/scheduler/context/og/{OGBFSAContext.java => OGSAContext.java} (73%) rename state-engine/src/main/java/scheduler/struct/{op => }/AbstractOperation.java (98%) delete mode 100644 state-engine/src/main/java/scheduler/struct/og/AbstractOperation.java create mode 100644 state-engine/src/main/java/scheduler/struct/og/Operation.java delete mode 100644 state-engine/src/main/java/scheduler/struct/og/nonstructured/AbstractNSOperationChain.java delete mode 100644 state-engine/src/main/java/scheduler/struct/og/nonstructured/NSAOperation.java delete mode 100644 state-engine/src/main/java/scheduler/struct/og/nonstructured/NSAOperationChain.java delete mode 100644 state-engine/src/main/java/scheduler/struct/og/nonstructured/NSOperation.java delete mode 100644 state-engine/src/main/java/scheduler/struct/og/nonstructured/NSOperationChain.java delete mode 100644 state-engine/src/main/java/scheduler/struct/og/structured/SOperationChain.java delete mode 100644 state-engine/src/main/java/scheduler/struct/og/structured/bfs/BFSOperation.java delete mode 100644 state-engine/src/main/java/scheduler/struct/og/structured/bfs/BFSOperationChain.java delete mode 100644 state-engine/src/main/java/scheduler/struct/og/structured/dfs/DFSOperation.java delete mode 100644 state-engine/src/main/java/scheduler/struct/og/structured/dfs/DFSOperationChain.java diff --git a/scripts/draw/model/model_exploration_strategy_batch.py b/scripts/draw/model/model_exploration_strategy_batch.py new file mode 100644 index 000000000..da56040ed --- /dev/null +++ b/scripts/draw/model/model_exploration_strategy_batch.py @@ -0,0 +1,209 @@ +import getopt +import os +import sys + +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import pylab +from matplotlib.font_manager import FontProperties +from matplotlib.ticker import LinearLocator, LogLocator, MaxNLocator +from numpy import double + +OPT_FONT_NAME = 'Helvetica' +TICK_FONT_SIZE = 24 +LABEL_FONT_SIZE = 28 +LEGEND_FONT_SIZE = 30 +LABEL_FP = FontProperties(style='normal', size=LABEL_FONT_SIZE) +LEGEND_FP = FontProperties(style='normal', size=LEGEND_FONT_SIZE) +TICK_FP = FontProperties(style='normal', size=TICK_FONT_SIZE) + +MARKERS = (['o', 's', 'v', "^", "h", "v", ">", "x", "d", "<", "|", "", "|", "_"]) +# you may want to change the color map for different figures +COLOR_MAP = ('#B03A2E', '#2874A6', '#239B56', '#7D3C98', '#F1C40F', '#F5CBA7', '#82E0AA', '#AEB6BF', '#AA4499') +# you may want to change the patterns for different figures +PATTERNS = (["\\", "///", "o", "||", "\\\\", "\\\\", "//////", "//////", ".", "\\\\\\", "\\\\\\"]) +LABEL_WEIGHT = 'bold' +LINE_COLORS = COLOR_MAP +LINE_WIDTH = 3.0 +MARKER_SIZE = 10.0 +MARKER_FREQUENCY = 1000 + +matplotlib.rcParams['ps.useafm'] = True +matplotlib.rcParams['pdf.use14corefonts'] = True +matplotlib.rcParams['xtick.labelsize'] = TICK_FONT_SIZE +matplotlib.rcParams['ytick.labelsize'] = TICK_FONT_SIZE +matplotlib.rcParams['font.family'] = OPT_FONT_NAME + +FIGURE_FOLDER = './results/model/exploration' +FILE_FOLER = '/home/shuhao/data/stats' + + +def ConvertEpsToPdf(dir_filename): + os.system("epstopdf --outfile " + dir_filename + ".pdf " + dir_filename + ".eps") + os.system("rm -rf " + dir_filename + ".eps") + + +# draw a bar chart +def DrawFigure(x_values, y_values, legend_labels, x_label, y_label, y_min, y_max, filename, allow_legend): + # you may change the figure size on your own. + fig = plt.figure(figsize=(10, 5)) + figure = fig.add_subplot(111) + + FIGURE_LABEL = legend_labels + + if not os.path.exists(FIGURE_FOLDER): + os.makedirs(FIGURE_FOLDER) + + # values in the x_xis + index = np.arange(len(x_values)) + # the bar width. + # you may need to tune it to get the best figure. + width = 0.08 + # draw the bars + bars = [None] * (len(FIGURE_LABEL)) + for i in range(len(y_values)): + bars[i] = plt.bar(index + i * width + width / 2, + y_values[i], width, + hatch=PATTERNS[i], + color=LINE_COLORS[i], + label=FIGURE_LABEL[i], + edgecolor="black", lw=3 + ) + + # sometimes you may not want to draw legends. + if allow_legend == True: + plt.legend(bars, FIGURE_LABEL, + prop=LEGEND_FP, + ncol=4, + loc='upper center', + # mode='expand', + shadow=False, + bbox_to_anchor=(0.5, 1.2), + columnspacing=0.1, + handletextpad=0.2, + # bbox_transform=ax.transAxes, + # frameon=True, + # columnspacing=5.5, + handlelength=2, + ) + + plt.xticks(index + 2 * width, x_values) + figure.get_xaxis().set_tick_params(direction='in', pad=10) + figure.get_yaxis().set_tick_params(direction='in', pad=10) + + plt.xlabel(x_label, fontproperties=LABEL_FP) + plt.ylabel(y_label, fontproperties=LABEL_FP) + + plt.savefig(FIGURE_FOLDER + "/" + filename + ".pdf", bbox_inches='tight') + + +def ReadFileGS(x_axis, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): + w, h = 2, len(x_axis) + y = [[] for _ in range(w)] + + for batchInterval in x_axis: + events = tthread * batchInterval + op_gs_path = getPathGS("OP_NS", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) + lines = open(op_gs_path).readlines() + throughput = lines[0].split(": ")[1] + y[0].append(float(throughput)) + + for batchInterval in x_axis: + events = tthread * batchInterval + op_dfs_path = getPathGS("OP_BFS", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) + lines = open(op_dfs_path).readlines() + throughput = lines[0].split(": ")[1] + y[1].append(float(throughput)) + + print(y) + return y + + +def ReadFileSL(x_axis, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): + w, h = 2, len(x_axis) + y = [[] for _ in range(w)] + + for batchInterval in x_axis: + events = tthread * batchInterval + op_gs_path = getPathSL("OP_NS", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) + lines = open(op_gs_path).readlines() + throughput = lines[0].split(": ")[1] + y[0].append(float(throughput)) + + for batchInterval in x_axis: + events = tthread * batchInterval + op_dfs_path = getPathSL("OP_BFS", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) + lines = open(op_dfs_path).readlines() + throughput = lines[0].split(": ")[1] + y[1].append(float(throughput)) + + print(y) + return y + + +def getPathSL(algo, events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): + return FILE_FOLER + '/StreamLedger/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}_{}'\ + .format(algo, tthread, events, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) + + +def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): + return FILE_FOLER + '/GrepSum/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}_{}'\ + .format(algo, tthread, events, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) + + +if __name__ == '__main__': + tthread = 24 + NUM_ITEMS = 115200 + NUM_ACCESS = 10 + deposit_ratio = 25 + key_skewness = 0 + overlap_ratio = 0 + abort_ratio = 0 + batchInterval = 10240 + isCyclic = "false" + complexity = 1000 + txn_length = 1 + + try: + opts, args = getopt.getopt(sys.argv[1:], "i:d:n:k:o:a:b:c:m:") + except getopt.GetoptError: + print("Error") + + for opt, arg in opts: + if opt in ['-i']: + NUM_ITEMS = int(arg) + elif opt in ['-d']: + deposit_ratio = int(arg) + elif opt in ['-n']: + NUM_ACCESS = int(arg) + elif opt in ['-k']: + key_skewness = int(arg) + elif opt in ['-o']: + overlap_ratio = int(arg) + elif opt in ['-a']: + abort_ratio = int(arg) + elif opt in ['-b']: + batchInterval = int(arg) + elif opt in ['-c']: + if int(arg) == 1: + isCyclic = "true" + else: + isCyclic = "false" + elif opt in ['-m']: + complexity = int(arg) + + x_axis = [5120, 10240, 20480, 40960, 81920] + # x_values = [int((x*tthread)) / NUM_ITEMS for x in x_axis] + x_values = x_axis + + legend_labels = ["NS", "BFS"] + legend = True + y_axis = ReadFileSL(x_axis, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) + DrawFigure(x_values, y_axis, legend_labels, "batch_size / keys", "Throughput (K/sec)", 0, 400, "sl_exploration_strategy_batch_t{}_b{}_{}_{}_{}_{}_{}_{}_{}" + .format(tthread, NUM_ITEMS, batchInterval, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity), + legend) + y_axis = ReadFileGS(x_axis, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) + DrawFigure(x_values, y_axis, legend_labels, "Punctuation Interval", "Throughput (K/sec)", 0, 400, "gs_exploration_strategy_batch_t{}_b{}_{}_{}_{}_{}_{}_{}_{}" + .format(tthread, NUM_ITEMS, batchInterval, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity), + legend) diff --git a/scripts/draw/model/model_granularity_access.py b/scripts/draw/model/model_granularity_access.py index a132d2d52..378a1b541 100644 --- a/scripts/draw/model/model_granularity_access.py +++ b/scripts/draw/model/model_granularity_access.py @@ -36,7 +36,7 @@ matplotlib.rcParams['font.family'] = OPT_FONT_NAME FIGURE_FOLDER = './results/model/granularity' -FILE_FOLER = '/home/shuhao/TStream/data/stats' +FILE_FOLER = '/home/shuhao/data/stats' def ConvertEpsToPdf(dir_filename): @@ -97,20 +97,20 @@ def DrawFigure(x_values, y_values, legend_labels, x_label, y_label, y_min, y_max plt.savefig(FIGURE_FOLDER + "/" + filename + ".pdf", bbox_inches='tight') -def ReadFileGS(x_axis, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): +def ReadFileGS(x_axis, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): w, h = 2, len(x_axis) y = [[] for _ in range(w)] for NUM_ACCESS in x_axis: events = tthread * batchInterval - op_gs_path = getPathGS("OP_NS_A", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + op_gs_path = getPathGS("OP_NS_A", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) lines = open(op_gs_path).readlines() throughput = lines[0].split(": ")[1] y[0].append(float(throughput)) for NUM_ACCESS in x_axis: events = tthread * batchInterval - op_gs_path = getPathGS("OG_NS_A", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + op_gs_path = getPathGS("OG_NS_A", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) lines = open(op_gs_path).readlines() throughput = lines[0].split(": ")[1] y[1].append(float(throughput)) @@ -120,14 +120,14 @@ def ReadFileGS(x_axis, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewne return y -def getPathSL(algo, events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): - return FILE_FOLER + '/StreamLedger/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}'\ - .format(algo, tthread, events, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) +def getPathSL(algo, events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): + return FILE_FOLER + '/StreamLedger/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}_{}'\ + .format(algo, tthread, events, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) -def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): - return FILE_FOLER + '/GrepSum/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}'\ - .format(algo, tthread, events, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) +def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): + return FILE_FOLER + '/GrepSum/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}_{}'\ + .format(algo, tthread, events, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) if __name__ == '__main__': @@ -141,6 +141,7 @@ def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overla batchInterval = 10240 isCyclic = "false" complexity = 1000 + txn_length = 1 try: opts, args = getopt.getopt(sys.argv[1:], "i:d:n:k:o:a:b:c:m:") @@ -175,8 +176,8 @@ def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overla legend_labels = ["Fine-grained", "Coarse-grained"] legend = True y_values = ReadFileGS(x_values, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, - abort_ratio, isCyclic, complexity) + abort_ratio, txn_length, isCyclic, complexity) DrawFigure(x_values, y_values, legend_labels, 'Num. of State Access Per Op.', 'Throughput (K/sec)', 0, - 400, 'granularity_comparison_access_t{}_b{}_{}_{}_{}_{}_{}_{}_{}' - .format(tthread, NUM_ITEMS, batchInterval, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity), legend) + 400, 'granularity_comparison_access_t{}_b{}_{}_{}_{}_{}_{}_{}_{}_{}' + .format(tthread, NUM_ITEMS, batchInterval, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity), legend) diff --git a/scripts/draw/model/model_granularity_batch.py b/scripts/draw/model/model_granularity_batch.py index 8a424d639..b717a4d9c 100644 --- a/scripts/draw/model/model_granularity_batch.py +++ b/scripts/draw/model/model_granularity_batch.py @@ -36,7 +36,7 @@ matplotlib.rcParams['font.family'] = OPT_FONT_NAME FIGURE_FOLDER = './results/model/granularity' -FILE_FOLER = '/home/shuhao/TStream/data/stats' +FILE_FOLER = '/home/shuhao/data/stats' def ConvertEpsToPdf(dir_filename): @@ -97,20 +97,20 @@ def DrawFigure(x_values, y_values, legend_labels, x_label, y_label, y_min, y_max plt.savefig(FIGURE_FOLDER + "/" + filename + ".pdf", bbox_inches='tight') -def ReadFileGS(x_axis, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): +def ReadFileGS(x_axis, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): w, h = 2, len(x_axis) y = [[] for _ in range(w)] for batchInterval in x_axis: events = tthread * batchInterval - op_gs_path = getPathGS("OP_NS_A", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + op_gs_path = getPathGS("OP_NS_A", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) lines = open(op_gs_path).readlines() throughput = lines[0].split(": ")[1] y[0].append(float(throughput)) for batchInterval in x_axis: events = tthread * batchInterval - op_gs_path = getPathGS("OG_NS_A", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + op_gs_path = getPathGS("OG_NS_A", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) lines = open(op_gs_path).readlines() throughput = lines[0].split(": ")[1] y[1].append(float(throughput)) @@ -120,20 +120,20 @@ def ReadFileGS(x_axis, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewne return y -def ReadFileSL(x_axis, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): +def ReadFileSL(x_axis, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): w, h = 2, len(x_axis) y = [[] for _ in range(w)] for batchInterval in x_axis: events = tthread * batchInterval - op_gs_path = getPathSL("OP_NS_A", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + op_gs_path = getPathSL("OP_NS_A", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) lines = open(op_gs_path).readlines() throughput = lines[0].split(": ")[1] y[0].append(float(throughput)) for batchInterval in x_axis: events = tthread * batchInterval - op_gs_path = getPathSL("OG_NS_A", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + op_gs_path = getPathSL("OG_NS_A", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) lines = open(op_gs_path).readlines() throughput = lines[0].split(": ")[1] y[1].append(float(throughput)) @@ -142,14 +142,14 @@ def ReadFileSL(x_axis, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_ske return y -def getPathSL(algo, events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): - return FILE_FOLER + '/StreamLedger/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}'\ - .format(algo, tthread, events, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) +def getPathSL(algo, events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): + return FILE_FOLER + '/StreamLedger/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}_{}'\ + .format(algo, tthread, events, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) -def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): - return FILE_FOLER + '/GrepSum/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}'\ - .format(algo, tthread, events, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) +def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): + return FILE_FOLER + '/GrepSum/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}_{}'\ + .format(algo, tthread, events, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) if __name__ == '__main__': @@ -163,6 +163,8 @@ def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overla batchInterval = 10240 isCyclic = "false" complexity = 1000 + txn_length = 1 + try: opts, args = getopt.getopt(sys.argv[1:], "i:d:n:k:o:a:b:c:m:") @@ -195,21 +197,22 @@ def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overla # Total Events / num_items # x_axis = [512, 1024, 2048, 4096, 5120, 10240, 20480, 40960] x_axis = [5120, 10240, 20480, 40960, 81920] - x_values = [int((x*tthread)) / NUM_ITEMS for x in x_axis] + # x_values = [int((x*tthread)) / NUM_ITEMS for x in x_axis] + x_values = x_axis legend_labels = ["Fine-grained", "Coarse-grained"] legend = True y_values = ReadFileGS(x_axis, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, - abort_ratio, isCyclic, complexity) + abort_ratio, txn_length, isCyclic, complexity) DrawFigure(x_values, y_values, legend_labels, 'Punctuation Interval', 'Throughput (K/sec)', 0, - 400, 'gs_granularity_comparison_batch_t{}_b{}_{}_{}_{}_{}_{}_{}_{}' - .format(tthread, NUM_ITEMS, batchInterval, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity), legend) + 400, 'gs_granularity_comparison_batch_t{}_b{}_{}_{}_{}_{}_{}_{}_{}_{}' + .format(tthread, NUM_ITEMS, batchInterval, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity), legend) y_values = ReadFileSL(x_axis, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, - abort_ratio, isCyclic, complexity) + abort_ratio, txn_length, isCyclic, complexity) DrawFigure(x_values, y_values, legend_labels, 'Punctuation Interval', 'Throughput (K/sec)', 0, - 400, 'sl_granularity_comparison_batch_t{}_b{}_{}_{}_{}_{}_{}_{}_{}' - .format(tthread, NUM_ITEMS, batchInterval, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity), legend) + 400, 'sl_granularity_comparison_batch_t{}_b{}_{}_{}_{}_{}_{}_{}_{}_{}' + .format(tthread, NUM_ITEMS, batchInterval, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity), legend) diff --git a/scripts/draw/model/model_granularity_cyclic.py b/scripts/draw/model/model_granularity_cyclic.py index 15309faa2..6caf65132 100644 --- a/scripts/draw/model/model_granularity_cyclic.py +++ b/scripts/draw/model/model_granularity_cyclic.py @@ -36,7 +36,7 @@ matplotlib.rcParams['font.family'] = OPT_FONT_NAME FIGURE_FOLDER = './results/model/granularity' -FILE_FOLER = '/home/shuhao/TStream/data/stats' +FILE_FOLER = '/home/shuhao/data/stats' def ConvertEpsToPdf(dir_filename): @@ -108,20 +108,20 @@ def DrawFigure(x_values, y_values, legend_labels, x_label, y_label, y_min, y_max plt.savefig(FIGURE_FOLDER + "/" + filename + ".pdf", bbox_inches='tight') -def ReadFileGS(x_axis, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): +def ReadFileGS(x_axis, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): w, h = 2, len(x_axis) y = [[] for _ in range(w)] for isCyclic in ["true", "false"]: events = tthread * batchInterval - op_gs_path = getPathGS("OP_NS_A", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + op_gs_path = getPathGS("OP_NS_A", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) lines = open(op_gs_path).readlines() throughput = lines[0].split(": ")[1] y[0].append(float(throughput)) for isCyclic in ["true", "false"]: events = tthread * batchInterval - op_gs_path = getPathGS("OG_NS_A", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + op_gs_path = getPathGS("OG_NS_A", events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) lines = open(op_gs_path).readlines() throughput = lines[0].split(": ")[1] y[1].append(float(throughput)) @@ -131,20 +131,20 @@ def ReadFileGS(x_axis, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewne return y -def ReadFileSL(x_axis, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): +def ReadFileSL(x_axis, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): w, h = 2, len(x_axis) y = [[] for _ in range(w)] for isCyclic in ["true", "false"]: events = tthread * batchInterval - op_gs_path = getPathSL("OP_NS_A", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + op_gs_path = getPathSL("OP_NS_A", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) lines = open(op_gs_path).readlines() throughput = lines[0].split(": ")[1] y[0].append(float(throughput)) for isCyclic in ["true", "false"]: events = tthread * batchInterval - op_gs_path = getPathSL("OG_NS_A", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + op_gs_path = getPathSL("OG_NS_A", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) lines = open(op_gs_path).readlines() throughput = lines[0].split(": ")[1] y[1].append(float(throughput)) @@ -154,14 +154,14 @@ def ReadFileSL(x_axis, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_ske return y -def getPathSL(algo, events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): - return FILE_FOLER + '/StreamLedger/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}'\ - .format(algo, tthread, events, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) +def getPathSL(algo, events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): + return FILE_FOLER + '/StreamLedger/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}_{}'\ + .format(algo, tthread, events, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) -def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): - return FILE_FOLER + '/GrepSum/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}'\ - .format(algo, tthread, events, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) +def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): + return FILE_FOLER + '/GrepSum/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}_{}'\ + .format(algo, tthread, events, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) if __name__ == '__main__': @@ -175,6 +175,7 @@ def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overla batchInterval = 10240 isCyclic = "false" complexity = 1000 + txn_length = 1 try: opts, args = getopt.getopt(sys.argv[1:], "i:d:n:k:o:a:b:c:m:") @@ -208,13 +209,13 @@ def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overla legend_labels = ["Single Op.", "Group of Op."] legend = True - y_values = ReadFileGS(x_values, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + y_values = ReadFileGS(x_values, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) DrawFigure(x_values, y_values, legend_labels, '', 'Throughput (K/sec)', 0, - 400, 'gs_granularity_comparison_cyclic_t{}_b{}_{}_{}_{}_{}_{}_{}_{}' - .format(tthread, NUM_ITEMS, batchInterval, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity), legend) - y_values = ReadFileSL(x_values, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + 400, 'gs_granularity_comparison_cyclic_t{}_b{}_{}_{}_{}_{}_{}_{}_{}_{}' + .format(tthread, NUM_ITEMS, batchInterval, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity), legend) + y_values = ReadFileSL(x_values, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) DrawFigure(x_values, y_values, legend_labels, '', 'Throughput (K/sec)', 0, - 400, 'sl_granularity_comparison_cyclic_t{}_b{}_{}_{}_{}_{}_{}_{}_{}' - .format(tthread, NUM_ITEMS, batchInterval, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity), legend) \ No newline at end of file + 400, 'sl_granularity_comparison_cyclic_t{}_b{}_{}_{}_{}_{}_{}_{}_{}_{}' + .format(tthread, NUM_ITEMS, batchInterval, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity), legend) \ No newline at end of file diff --git a/scripts/draw/model/model_granularity_writeonly.py b/scripts/draw/model/model_granularity_writeonly.py index 56a31b5cc..f930dadf7 100644 --- a/scripts/draw/model/model_granularity_writeonly.py +++ b/scripts/draw/model/model_granularity_writeonly.py @@ -36,7 +36,7 @@ matplotlib.rcParams['font.family'] = OPT_FONT_NAME FIGURE_FOLDER = './results/model/granularity' -FILE_FOLER = '/home/shuhao/TStream/data/stats' +FILE_FOLER = '/home/shuhao/data/stats' def ConvertEpsToPdf(dir_filename): @@ -97,20 +97,20 @@ def DrawFigure(x_values, y_values, legend_labels, x_label, y_label, y_min, y_max plt.savefig(FIGURE_FOLDER + "/" + filename + ".pdf", bbox_inches='tight') -def ReadFileSL(x_axis, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): +def ReadFileSL(x_axis, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): w, h = 2, len(x_axis) y = [[] for _ in range(w)] for deposit_ratio in x_axis: events = tthread * batchInterval - op_gs_path = getPathSL("OPGSA", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + op_gs_path = getPathSL("OP_NS_A", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) lines = open(op_gs_path).readlines() throughput = lines[0].split(": ")[1] y[0].append(float(throughput)) for deposit_ratio in x_axis: events = tthread * batchInterval - op_gs_path = getPathSL("GSA", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) + op_gs_path = getPathSL("OG_NS_A", events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) lines = open(op_gs_path).readlines() throughput = lines[0].split(": ")[1] y[1].append(float(throughput)) @@ -119,14 +119,14 @@ def ReadFileSL(x_axis, tthread, batchInterval, NUM_ITEMS, deposit_ratio, key_ske return y -def getPathSL(algo, events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): - return FILE_FOLER + '/StreamLedger/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}'\ - .format(algo, tthread, events, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) +def getPathSL(algo, events, tthread, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): + return FILE_FOLER + '/StreamLedger/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}_{}'\ + .format(algo, tthread, events, NUM_ITEMS, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) -def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity): - return FILE_FOLER + '/GrepSum/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}'\ - .format(algo, tthread, events, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity) +def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity): + return FILE_FOLER + '/GrepSum/{}/threads = {}/totalEvents = {}/{}_{}_{}_{}_{}_{}_{}_{}'\ + .format(algo, tthread, events, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity) if __name__ == '__main__': @@ -140,6 +140,7 @@ def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overla batchInterval = 10240 isCyclic = "false" complexity = 1000 + txn_length = 1 try: opts, args = getopt.getopt(sys.argv[1:], "i:d:n:k:o:a:b:c:m:") @@ -174,8 +175,8 @@ def getPathGS(algo, events, tthread, NUM_ITEMS, NUM_ACCESS, key_skewness, overla legend_labels = ["Fine-grained", "Coarse-grained"] legend = True y_values = ReadFileSL(x_values, tthread, batchInterval, NUM_ITEMS, NUM_ACCESS, key_skewness, overlap_ratio, - abort_ratio, isCyclic, complexity) + abort_ratio, txn_length, isCyclic, complexity) DrawFigure(x_values, y_values, legend_labels, 'Ratio of Writeonly Txns', 'Throughput (K/sec)', 0, - 400, 'sl_granularity_comparison_writeonly_t{}_b{}_{}_{}_{}_{}_{}_{}_{}' - .format(tthread, NUM_ITEMS, batchInterval, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, isCyclic, complexity), legend) + 400, 'sl_granularity_comparison_writeonly_t{}_b{}_{}_{}_{}_{}_{}_{}_{}_{}' + .format(tthread, NUM_ITEMS, batchInterval, deposit_ratio, key_skewness, overlap_ratio, abort_ratio, txn_length, isCyclic, complexity), legend) diff --git a/scripts/model_runner.sh b/scripts/model_runner.sh index 14d1486e7..ada4979d0 100644 --- a/scripts/model_runner.sh +++ b/scripts/model_runner.sh @@ -7,7 +7,7 @@ function ResetParameters() { checkpointInterval=10240 tthread=24 scheduler="BFS" - deposit_ratio=25 + deposit_ratio=50 key_skewness=0 overlap_ratio=0 abort_ratio=0 @@ -70,34 +70,47 @@ function withAbortEvaluation() { function granularity_study() { # isCyclic - ResetParameters - for app in GrepSum StreamLedger - do - # for tthread in 24 - for isCyclic in 0 1 - do - withAbortEvaluation - done - done + # ResetParameters + # deposit_ratio=75 + # for app in GrepSum StreamLedger + # do + # # for tthread in 24 + # for isCyclic in 0 1 + # do + # withAbortEvaluation + # done + # done - # Num of PD - ResetParameters - for app in GrepSum - do - # for tthread in 24 - for isCyclic in 0 - do - for NUM_ACCESS in 1 2 4 6 8 10 - do - withAbortEvaluation - done - done - done + # # Num of PD + # ResetParameters + # for app in GrepSum + # do + # # for tthread in 24 + # for isCyclic in 0 + # do + # for NUM_ACCESS in 1 2 4 6 8 10 + # do + # withAbortEvaluation + # done + # done + # done + # ResetParameters + # for app in StreamLedger + # do + # # for tthread in 24 + # for isCyclic in 0 + # do + # for deposit_ratio in 0 25 50 75 100 + # do + # withAbortEvaluation + # done + # done + # done # num of TD ResetParameters NUM_ACCESS=1 # OC level and OP level has similar performance before - deposit_ratio=90 # try to avoid the overhead of PD + deposit_ratio=100 # try to avoid the overhead of PD for app in StreamLedger GrepSum do # for tthread in 24 @@ -109,20 +122,110 @@ function granularity_study() { done done done +} - # Write Only Ratio +function exploration_strategy_study() { +# # Average num of OP in OC +# ResetParameters +# for app in StreamLedger GrepSum +# do +# # for tthread in 24 +# for isCyclic in 1 +# do +# for complexity in 0 10000 20000 40000 60000 80000 100000 +# do +# for scheduler in OPDFS OPBFS OPGS +# do +# runTStream +# done +# done +# done +# done + +# # Average num of OP in OC +# ResetParameters +# for app in StreamLedger GrepSum +# do +# # for tthread in 24 +# for isCyclic in 1 +# do +# for NUM_ITEMS in 12288 122880 1228800 +# do +# for scheduler in OPDFS OPBFS OPGS +# do +# runTStream +# done +# done +# done +# done + + # Average num of OP in OC ResetParameters - for app in StreamLedger + for app in StreamLedger GrepSum do # for tthread in 24 for isCyclic in 0 do - for deposit_ratio in 0 25 50 75 100 + for checkpointInterval in 5120 10240 20480 40960 81920 do - withAbortEvaluation + for scheduler in OP_BFS OP_NS + do + runTStream + done done done done + + # # Write Only Ratio + # ResetParameters + # for app in StreamLedger + # do + # # for tthread in 24 + # for isCyclic in 1 + # do + # for deposit_ratio in 0 25 50 75 100 + # do + # for scheduler in OPDFS OPBFS OPGS + # do + # runTStream + # done + # done + # done + # done + + # # Num of FD + # ResetParameters + # for app in GrepSum + # do + # # for tthread in 24 + # for isCyclic in 1 + # do + # for NUM_ACCESS in 1 2 4 6 8 10 + # do + # for scheduler in OPDFS OPBFS OPGS + # do + # runTStream + # done + # done + # done + # done + + # # key_skewness + # ResetParameters + # for app in GrepSum StreamLedger + # do + # # for tthread in 24 + # for isCyclic in 1 + # do + # for key_skewness in 0 25 50 75 100 + # do + # for scheduler in OPDFS OPBFS OPGS + # do + # runTStream + # done + # done + # done + # done } function abort_mechanism_study() { @@ -215,158 +318,52 @@ function abort_mechanism_study() { done } -function exploration_strategy_study() { -# Average num of OP in OC - ResetParameters - for app in StreamLedger GrepSum - do - # for tthread in 24 - for isCyclic in 1 - do - for complexity in 0 10000 20000 40000 60000 80000 100000 - do - for scheduler in OPDFS OPBFS OPGS - do - runTStream - done - done - done - done - - # Average num of OP in OC - ResetParameters - for app in StreamLedger GrepSum - do - # for tthread in 24 - for isCyclic in 1 - do - for NUM_ITEMS in 12288 122880 1228800 - do - for scheduler in OPDFS OPBFS OPGS - do - runTStream - done - done - done - done - - # Average num of OP in OC - ResetParameters - for app in StreamLedger GrepSum - do - # for tthread in 24 - for isCyclic in 1 - do - for checkpointInterval in 5120 10240 20480 40960 81920 - do - for scheduler in OPDFS OPBFS OPGS - do - runTStream - done - done - done - done - - # Write Only Ratio - ResetParameters - for app in StreamLedger - do - # for tthread in 24 - for isCyclic in 1 - do - for deposit_ratio in 0 25 50 75 100 - do - for scheduler in OPDFS OPBFS OPGS - do - runTStream - done - done - done - done - - # Num of FD - ResetParameters - for app in GrepSum - do - # for tthread in 24 - for isCyclic in 1 - do - for NUM_ACCESS in 1 2 4 6 8 10 - do - for scheduler in OPDFS OPBFS OPGS - do - runTStream - done - done - done - done - - # key_skewness - ResetParameters - for app in GrepSum StreamLedger - do - # for tthread in 24 - for isCyclic in 1 - do - for key_skewness in 0 25 50 75 100 - do - for scheduler in OPDFS OPBFS OPGS - do - runTStream - done - done - done - done -} - # Granularity selection -granularity_study -## Exploration Strategy -#exploration_strategy_study +# granularity_study +# Exploration Strategy +# exploration_strategy_study ## Abort mechanism selection #abort_mechanism_study -cd draw || exit +# cd draw || exit -ResetParameters -echo "python model/model_granularity_cyclic.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" - python model/model_granularity_cyclic.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity +# ResetParameters +# NUM_ACCESS=1 # OC level and OP level has similar performance before +# deposit_ratio=100 # try to avoid the overhead of PD +# echo "python model/model_granularity_cyclic.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" +# python model/model_granularity_cyclic.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity -#NUM_ACCESS=1 # where OP and OC has the similar performance under default setting -# cd draw || exit -echo "python model/model_granularity_access.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" -python model/model_granularity_access.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity +# #NUM_ACCESS=1 # where OP and OC has the similar performance under default setting +# # cd draw || exit +# echo "python model/model_granularity_access.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" +# python model/model_granularity_access.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity -echo "python model/model_granularity_batch.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" -python model/model_granularity_batch.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity +# echo "python model/model_granularity_batch.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" +# python model/model_granularity_batch.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity -echo "python model/model_granularity_writeonly.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" -python model/model_granularity_writeonly.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity +# echo "python model/model_granularity_writeonly.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" +# python model/model_granularity_writeonly.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity -#ResetParameters -#abort_ratio=0 -## cd draw || exit -#for isCyclic in 1 -#do +ResetParameters +cd draw || exit # echo "python model_exploration_strategy_key.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" # python model_exploration_strategy_key.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity -# -# echo "python model_exploration_strategy_batch.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" -# python model_exploration_strategy_batch.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity -# + +echo "model/python model_exploration_strategy_batch.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" +python model/model_exploration_strategy_batch.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity + # echo "python model_exploration_strategy_access.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" # python model_exploration_strategy_access.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity -# + # echo "python model_exploration_strategy_writeonly.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" # python model_exploration_strategy_writeonly.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity -# + # echo "python model_exploration_strategy_skewness.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" # python model_exploration_strategy_skewness.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity -# + # echo "python model_exploration_strategy_complexity.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" # python model_exploration_strategy_complexity.py -i $NUM_ITEMS -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity -#done # # #ResetParameters diff --git a/scripts/runner.sh b/scripts/runner.sh new file mode 100644 index 000000000..83e0a991e --- /dev/null +++ b/scripts/runner.sh @@ -0,0 +1,197 @@ +#!/bin/bash + +function ResetParameters() { + app="StreamLedger" + NUM_ITEMS=122880 + NUM_ACCESS=10 + # checkpointInterval=245760 + tthread=24 + scheduler="BFS" + deposit_ratio=25 + key_skewness=25 + overlap_ratio=0 + abort_ratio=100 + CCOption=3 #TSTREAM + complexity=10000 + isCyclic=1 + totalEvents=245760 +} + +function runTStream() { + checkpointInterval=`expr $totalEvents \* $tthread` + # NUM_ITEMS=`expr $totalEvents` + echo "java -Xms100g -Xmx100g -jar -d64 application-0.0.2-jar-with-dependencies.jar \ + --app $app \ + --NUM_ITEMS $NUM_ITEMS \ + --NUM_ACCESS $NUM_ACCESS \ + --tthread $tthread \ + --scheduler $scheduler \ + --totalEvents $totalEvents \ + --checkpoint_interval $checkpointInterval \ + --deposit_ratio $deposit_ratio \ + --key_skewness $key_skewness \ + --overlap_ratio $overlap_ratio \ + --abort_ratio $abort_ratio \ + --CCOption $CCOption \ + --complexity $complexity \ + --isCyclic $isCyclic" + java -Xms100g -Xmx100g -Xss100M -jar -d64 application-0.0.2-jar-with-dependencies.jar \ + --app $app \ + --NUM_ITEMS $NUM_ITEMS \ + --NUM_ACCESS $NUM_ACCESS \ + --tthread $tthread \ + --scheduler $scheduler \ + --totalEvents $totalEvents \ + --checkpoint_interval $checkpointInterval \ + --deposit_ratio $deposit_ratio \ + --key_skewness $key_skewness \ + --overlap_ratio $overlap_ratio \ + --abort_ratio $abort_ratio \ + --CCOption $CCOption \ + --complexity $complexity \ + --isCyclic $isCyclic +} + +# run basic experiment for different algorithms +function baselineEvaluation() { + # for scheduler in BFS DFS GS OPBFS OPDFS OPGS TStream + for scheduler in OPGSA TStream + # for scheduler in OPGSA + do + runTStream + done +} + + +function patEvluation() { + CCOption=4 #SSTORE + runTStream +} + +# overview experiment scalability exp +function overview_scalability() { + ResetParameters + # for app in StreamLedger GrepSum + for app in GrepSum + do + for isCyclic in 1 + do + for tthread in 1 4 8 12 16 20 24 + # for tthread in 24 + do + baselineEvaluation + done + done + done + + ResetParameters + # for app in StreamLedger GrepSum + for app in GrepSum + do + for isCyclic in 1 + do + for tthread in 1 4 8 12 16 20 24 + # for tthread in 24 + do + patEvluation + done + done + done +} + +function overview_cyclic() { + ResetParameters + for app in StreamLedger GrepSum + do + for isCyclic in 0 1 + do + for tthread in 24 + do + baselineEvaluation + done + done + done + + ResetParameters + for app in StreamLedger GrepSum + do + for isCyclic in 0 1 + do + for tthread in 24 + do + patEvluation + done + done + done +} + + +function overview() { # multi-batch exp + ResetParameters + for isCyclic in 1 + do + for NUM_ITEMS in 12288 122880 1228800 + do + for abort_ratio in 0 10 100 1000 5000 + do + for key_skewness in 0 25 50 75 100 + do + app=StreamLedger + for deposit_ratio in 0 25 50 75 100 + do + baselineEvaluation + done + app=GrepSum + for NUM_ACCESS in 0 2 4 6 8 10 + do + baselineEvaluation + done + done + done + done + done + + ResetParameters + for isCyclic in 1 + do + for NUM_ITEMS in 12288 122880 1228800 + do + for abort_ratio in 0 10 100 1000 5000 + do + for key_skewness in 0 25 50 75 100 + do + app=StreamLedger + for deposit_ratio in 0 25 50 75 100 + do + patEvluation + done + app=GrepSum + for NUM_ACCESS in 0 2 4 6 8 10 + do + patEvluation + done + done + done + done + done +} + + +overview_scalability +# overview_cyclic +ResetParameters +cd draw || exit +# echo "python sensitivity_cyclic.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" +# python sensitivity_cyclic.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity +echo "python overview_all.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" +python overview_all.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity +# for isCyclic in 0 1 +# do +# # python overview.py +# echo "python overview_all.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" +# python overview_all.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity +# echo "python overview_all_latency.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" +# python overview_all_latency.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity +# echo "python overview_breakdown.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity" +# python overview_breakdown.py -i $NUM_ITEMS -d $deposit_ratio -n $NUM_ACCESS -k $key_skewness -o $overlap_ratio -a $abort_ratio -b $checkpointInterval -c $isCyclic -m $complexity +# done diff --git a/state-engine/src/main/java/scheduler/context/og/AbstractOGNSContext.java b/state-engine/src/main/java/scheduler/context/og/AbstractOGNSContext.java index 94da1903a..32874373a 100644 --- a/state-engine/src/main/java/scheduler/context/og/AbstractOGNSContext.java +++ b/state-engine/src/main/java/scheduler/context/og/AbstractOGNSContext.java @@ -1,15 +1,15 @@ package scheduler.context.og; import scheduler.statemanager.og.OperationChainStateListener; -import scheduler.struct.og.nonstructured.AbstractNSOperationChain; -import scheduler.struct.og.nonstructured.NSOperation; +import scheduler.struct.og.OperationChain; import java.util.ArrayDeque; -public abstract class AbstractOGNSContext> extends OGSchedulerContext { +public abstract class AbstractOGNSContext extends OGSchedulerContext { - public ArrayDeque IsolatedOC; - public ArrayDeque OCwithChildren; + + public ArrayDeque IsolatedOC; + public ArrayDeque OCwithChildren; //TODO: Make it flexible to accept other applications. //The table name is hard-coded. @@ -33,17 +33,20 @@ public void redo() { OCwithChildren.clear(); } - @Override - public SchedulingUnit createTask(String tableName, String pKey, long bid) { - throw new UnsupportedOperationException("Unsupported."); - } - @Override public boolean finished() { assert scheduledOPs <= totalOsToSchedule; return scheduledOPs == totalOsToSchedule; } + @Override + public OperationChain createTask(String tableName, String pKey, long bid) { + OperationChain oc = new OperationChain(tableName, pKey, bid); + oc.setContext(this); +// operationChains.add(oc); + return oc; + } + public OperationChainStateListener getListener() { throw new UnsupportedOperationException(); } diff --git a/state-engine/src/main/java/scheduler/context/og/OGBFSContext.java b/state-engine/src/main/java/scheduler/context/og/OGBFSContext.java deleted file mode 100644 index 7c6cbd622..000000000 --- a/state-engine/src/main/java/scheduler/context/og/OGBFSContext.java +++ /dev/null @@ -1,20 +0,0 @@ -package scheduler.context.og; - -import scheduler.struct.og.structured.bfs.BFSOperation; -import scheduler.struct.og.structured.bfs.BFSOperationChain; - -public class OGBFSContext extends OGSContext { - - //TODO: Make it flexible to accept other applications. - //The table name is hard-coded. - public OGBFSContext(int thisThreadId, int totalThreads) { - super(thisThreadId, totalThreads); - } - - @Override - public BFSOperationChain createTask(String tableName, String pKey, long bid) { - BFSOperationChain oc = new BFSOperationChain(tableName, pKey, bid); -// operationChains.add(oc); - return oc; - } -} diff --git a/state-engine/src/main/java/scheduler/context/og/OGDFSAContext.java b/state-engine/src/main/java/scheduler/context/og/OGDFSAContext.java deleted file mode 100644 index 149b81118..000000000 --- a/state-engine/src/main/java/scheduler/context/og/OGDFSAContext.java +++ /dev/null @@ -1,12 +0,0 @@ -package scheduler.context.og; - -public class OGDFSAContext extends OGDFSContext { - - public int rollbackLevel = -1; // initialized to 0 if thread not required to be rollbacked. - public boolean isRollbacked = false; // initialized to 0 if thread not required to be rollbacked. - - //The table name is hard-coded. - public OGDFSAContext(int thisThreadId, int totalThreads) { - super(thisThreadId, totalThreads); - } -} diff --git a/state-engine/src/main/java/scheduler/context/og/OGDFSContext.java b/state-engine/src/main/java/scheduler/context/og/OGDFSContext.java deleted file mode 100644 index 1841fb2f8..000000000 --- a/state-engine/src/main/java/scheduler/context/og/OGDFSContext.java +++ /dev/null @@ -1,20 +0,0 @@ -package scheduler.context.og; - -import scheduler.struct.og.structured.dfs.DFSOperation; -import scheduler.struct.og.structured.dfs.DFSOperationChain; - -public class OGDFSContext extends OGSContext { - - //TODO: Make it flexible to accept other applications. - //The table name is hard-coded. - public OGDFSContext(int thisThreadId, int totalThreads) { - super(thisThreadId, totalThreads); - } - - @Override - public DFSOperationChain createTask(String tableName, String pKey, long bid) { - DFSOperationChain oc = new DFSOperationChain(tableName, pKey, bid); -// operationChains.add(oc); - return oc; - } -} diff --git a/state-engine/src/main/java/scheduler/context/og/OGNSAContext.java b/state-engine/src/main/java/scheduler/context/og/OGNSAContext.java index 72185f9b7..73f159403 100644 --- a/state-engine/src/main/java/scheduler/context/og/OGNSAContext.java +++ b/state-engine/src/main/java/scheduler/context/og/OGNSAContext.java @@ -3,14 +3,11 @@ import scheduler.impl.og.nonstructured.OGNSAScheduler; import scheduler.statemanager.og.OperationChainStateListener; import scheduler.statemanager.og.PartitionStateManagerWithAbort; -import scheduler.struct.og.nonstructured.NSAOperationChain; -import scheduler.struct.og.nonstructured.NSAOperation; -public class OGNSAContext extends AbstractOGNSContext { +public class OGNSAContext extends AbstractOGNSContext { public final PartitionStateManagerWithAbort partitionStateManager; - //TODO: Make it flexible to accept other applications. //The table name is hard-coded. public OGNSAContext(int thisThreadId, int totalThreads) { @@ -18,15 +15,6 @@ public OGNSAContext(int thisThreadId, int totalThreads) { partitionStateManager = new PartitionStateManagerWithAbort(); } - @Override - public NSAOperationChain createTask(String tableName, String pKey, long bid) { - NSAOperationChain oc = new NSAOperationChain(tableName, pKey, bid); - oc.setContext(this); -// if (!operationChainsLeft.contains(oc)) -// operationChains.add(oc); - return oc; - } - public void initialize(OGNSAScheduler.ExecutableTaskListener executableTaskListener) { partitionStateManager.initialize(executableTaskListener); } diff --git a/state-engine/src/main/java/scheduler/context/og/OGNSContext.java b/state-engine/src/main/java/scheduler/context/og/OGNSContext.java index 860435101..530acf9ae 100644 --- a/state-engine/src/main/java/scheduler/context/og/OGNSContext.java +++ b/state-engine/src/main/java/scheduler/context/og/OGNSContext.java @@ -3,13 +3,10 @@ import scheduler.impl.og.nonstructured.OGNSScheduler; import scheduler.statemanager.og.OperationChainStateListener; import scheduler.statemanager.og.PartitionStateManager; -import scheduler.struct.og.nonstructured.NSOperation; -import scheduler.struct.og.nonstructured.NSOperationChain; import java.util.ArrayDeque; -public class OGNSContext - extends AbstractOGNSContext { +public class OGNSContext extends AbstractOGNSContext { public final PartitionStateManager partitionStateManager; @@ -17,25 +14,16 @@ public class OGNSContext //The table name is hard-coded. public OGNSContext(int thisThreadId, int totalThreads) { super(thisThreadId); - partitionStateManager = new PartitionStateManager(); IsolatedOC = new ArrayDeque<>(); OCwithChildren = new ArrayDeque<>(); requests = new ArrayDeque<>(); - } - - @Override - public NSOperationChain createTask(String tableName, String pKey, long bid) { - NSOperationChain oc = new NSOperationChain(tableName, pKey, bid); - oc.setContext(this); -// operationChains.add(oc); - return oc; + partitionStateManager = new PartitionStateManager(); } public void initialize(OGNSScheduler.ExecutableTaskListener executableTaskListener) { partitionStateManager.initialize(executableTaskListener); } - @Override public OperationChainStateListener getListener() { return partitionStateManager; } diff --git a/state-engine/src/main/java/scheduler/context/og/OGBFSAContext.java b/state-engine/src/main/java/scheduler/context/og/OGSAContext.java similarity index 73% rename from state-engine/src/main/java/scheduler/context/og/OGBFSAContext.java rename to state-engine/src/main/java/scheduler/context/og/OGSAContext.java index 17c2370ee..c0188bd88 100644 --- a/state-engine/src/main/java/scheduler/context/og/OGBFSAContext.java +++ b/state-engine/src/main/java/scheduler/context/og/OGSAContext.java @@ -1,12 +1,12 @@ package scheduler.context.og; -public class OGBFSAContext extends OGBFSContext { +public class OGSAContext extends OGSContext { public int rollbackLevel = -1; // initialized to 0 if thread not required to be rollbacked. public boolean isRollbacked = false; // initialized to 0 if thread not required to be rollbacked. //The table name is hard-coded. - public OGBFSAContext(int thisThreadId, int totalThreads) { + public OGSAContext(int thisThreadId, int totalThreads) { super(thisThreadId, totalThreads); } } diff --git a/state-engine/src/main/java/scheduler/context/og/OGSContext.java b/state-engine/src/main/java/scheduler/context/og/OGSContext.java index c1b44a86b..2885aa487 100644 --- a/state-engine/src/main/java/scheduler/context/og/OGSContext.java +++ b/state-engine/src/main/java/scheduler/context/og/OGSContext.java @@ -1,19 +1,17 @@ package scheduler.context.og; -import scheduler.struct.og.AbstractOperation; import scheduler.struct.og.OperationChain; -import scheduler.struct.og.structured.SOperationChain; import java.util.*; -public abstract class OGSContext> extends OGSchedulerContext { +public class OGSContext extends OGSchedulerContext { - public HashMap> allocatedLayeredOCBucket;// + public HashMap> allocatedLayeredOCBucket;// public int currentLevel; public int currentLevelIndex; public int totalThreads; public int maxLevel;//total number of operations to process per thread. - public SchedulingUnit ready_oc;//ready operation chain per thread. + public OperationChain ready_oc;//ready operation chain per thread. //TODO: Make it flexible to accept other applications. //The table name is hard-coded. @@ -41,11 +39,13 @@ public void redo() { } @Override - public SchedulingUnit createTask(String tableName, String pKey, long bid) { - throw new UnsupportedOperationException("Unsupported."); + public OperationChain createTask(String tableName, String pKey, long bid) { + OperationChain oc = new OperationChain(tableName, pKey, bid); +// operationChains.add(oc); + return oc; } - public ArrayList OCSCurrentLayer() { + public ArrayList OCSCurrentLayer() { return allocatedLayeredOCBucket.get(currentLevel); } @@ -67,35 +67,35 @@ public boolean exploreFinished() { * * @param ocs */ - public void buildBucketPerThread(Collection ocs, HashSet> resolvedOC) { + public void buildBucketPerThread(Collection ocs, HashSet resolvedOC) { // TODO: update this logic to the latest logic that we proposed in operation level int localMaxDLevel = 0; int dependencyLevel; - for (SchedulingUnit oc : ocs) { + for (OperationChain oc : ocs) { if (oc.getOperations().isEmpty()) { continue; } this.totalOsToSchedule += oc.getOperations().size(); - oc.updateDependencyLevel(); + ((OperationChain) oc).updateDependencyLevel(); if (resolvedOC.contains(oc)) { continue; } - dependencyLevel = oc.getDependencyLevel(); + dependencyLevel = ((OperationChain) oc).getDependencyLevel(); if (localMaxDLevel < dependencyLevel) localMaxDLevel = dependencyLevel; if (!allocatedLayeredOCBucket.containsKey(dependencyLevel)) allocatedLayeredOCBucket.put(dependencyLevel, new ArrayList<>()); - allocatedLayeredOCBucket.get(dependencyLevel).add(oc); + allocatedLayeredOCBucket.get(dependencyLevel).add((OperationChain) oc); } // if (enable_log) LOG.debug("localMaxDLevel" + localMaxDLevel); this.maxLevel = localMaxDLevel; } - public void putBusyWaitOCs(HashSet resolvedOC, int maxLevel) { - for (SchedulingUnit oc : resolvedOC) { + public void putBusyWaitOCs(HashSet resolvedOC, int maxLevel) { + for (OperationChain oc : resolvedOC) { if (!allocatedLayeredOCBucket.containsKey(maxLevel)) allocatedLayeredOCBucket.put(maxLevel, new ArrayList<>()); - allocatedLayeredOCBucket.get(maxLevel).add(oc); + allocatedLayeredOCBucket.get(maxLevel).add((OperationChain) oc); } this.maxLevel = maxLevel; } diff --git a/state-engine/src/main/java/scheduler/context/og/OGSchedulerContext.java b/state-engine/src/main/java/scheduler/context/og/OGSchedulerContext.java index f24be2922..842987f69 100644 --- a/state-engine/src/main/java/scheduler/context/og/OGSchedulerContext.java +++ b/state-engine/src/main/java/scheduler/context/og/OGSchedulerContext.java @@ -2,17 +2,18 @@ import scheduler.Request; import scheduler.context.SchedulerContext; +import scheduler.struct.og.OperationChain; import java.util.*; -public abstract class OGSchedulerContext implements SchedulerContext { - public final ArrayDeque busyWaitQueue; +public abstract class OGSchedulerContext implements SchedulerContext { + public final ArrayDeque busyWaitQueue; public int thisThreadId; public ArrayDeque requests; public int scheduledOPs;//current number of operations processed per thread. public int totalOsToSchedule;//total number of operations to process per thread. - public Set operationChains = new HashSet<>(); + public Set operationChains = new HashSet<>(); public int fd = 0; @@ -40,5 +41,5 @@ public void push(Request request) { requests.push(request); } - public abstract SchedulingUnit createTask(String tableName, String pKey, long bid); + public abstract OperationChain createTask(String tableName, String pKey, long bid); } diff --git a/state-engine/src/main/java/scheduler/context/op/OPSContext.java b/state-engine/src/main/java/scheduler/context/op/OPSContext.java index 313310921..8878bc9a8 100644 --- a/state-engine/src/main/java/scheduler/context/op/OPSContext.java +++ b/state-engine/src/main/java/scheduler/context/op/OPSContext.java @@ -14,7 +14,7 @@ public class OPSContext extends OPSchedulerContext { public int currentLevelIndex; public int totalThreads; public int maxLevel;//total number of operations to process per thread. - public Operation ready_oc;//ready operation chain per thread. + public Operation ready_op;//ready operation per thread. public ConcurrentLinkedQueue layerBuildHelperQueue = new ConcurrentLinkedQueue<>(); public OPSContext(int thisThreadId) { diff --git a/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java b/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java index f4fdad90c..303e77c0b 100644 --- a/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java @@ -7,10 +7,10 @@ import scheduler.Request; import scheduler.context.og.OGSchedulerContext; import scheduler.impl.IScheduler; -import scheduler.struct.op.MetaTypes; -import scheduler.struct.og.AbstractOperation; +import scheduler.struct.og.Operation; import scheduler.struct.og.OperationChain; import scheduler.struct.og.TaskPrecedenceGraph; +import scheduler.struct.op.MetaTypes; import storage.SchemaRecord; import storage.TableRecord; import storage.datatype.DataBox; @@ -21,15 +21,16 @@ import utils.AppConfig; import utils.SOURCE_CONTROL; +import java.util.ArrayList; import java.util.List; import static content.common.CommonMetaTypes.AccessType.*; -public abstract class OGScheduler, ExecutionUnit extends AbstractOperation, SchedulingUnit extends OperationChain> +public abstract class OGScheduler implements IScheduler { private static final Logger log = LoggerFactory.getLogger(OGScheduler.class); public final int delta;//range of each partition. depends on the number of op in the stage. - public final TaskPrecedenceGraph tpg; // TPG to be maintained in this global instance. + public final TaskPrecedenceGraph tpg; // TPG to be maintained in this global instance. protected OGScheduler(int totalThreads, int NUM_ITEMS, int app) { delta = (int) Math.ceil(NUM_ITEMS / (double) totalThreads); // Check id generation in DateGenerator. @@ -87,7 +88,7 @@ public void start_evaluation(Context context, long mark_ID, int num_events) { * @param previous_mark_ID * @param clean */ - protected void Transfer_Fun(ExecutionUnit operation, long previous_mark_ID, boolean clean) { + protected void Transfer_Fun(Operation operation, long previous_mark_ID, boolean clean) { SchemaRecord preValues = operation.condition_records[0].content_.readPreValues(operation.bid); final long sourceAccountBalance = preValues.getValues().get(1).getLong(); @@ -126,7 +127,7 @@ protected void Transfer_Fun(ExecutionUnit operation, long previous_mark_ID, bool * @param mark_ID * @param clean */ - protected void Depo_Fun(ExecutionUnit operation, long mark_ID, boolean clean) { + protected void Depo_Fun(Operation operation, long mark_ID, boolean clean) { SchemaRecord srcRecord = operation.s_record.content_.readPreValues(operation.bid); List values = srcRecord.getValues(); AppConfig.randomDelay(); @@ -137,7 +138,7 @@ protected void Depo_Fun(ExecutionUnit operation, long mark_ID, boolean clean) { operation.s_record.content_.updateMultiValues(operation.bid, mark_ID, clean, tempo_record);//it may reduce NUMA-traffic. } - protected void GrepSum_Fun(ExecutionUnit operation, long previous_mark_ID, boolean clean) { + protected void GrepSum_Fun(Operation operation, long previous_mark_ID, boolean clean) { int keysLength = operation.condition_records.length; SchemaRecord[] preValues = new SchemaRecord[operation.condition_records.length]; @@ -178,7 +179,7 @@ protected void GrepSum_Fun(ExecutionUnit operation, long previous_mark_ID, boole * @param mark_ID * @param clean */ - public void execute(ExecutionUnit operation, long mark_ID, boolean clean) { + public void execute(Operation operation, long mark_ID, boolean clean) { if (operation.getOperationState().equals(MetaTypes.OperationStateType.ABORTED)) { return; // return if the operation is already aborted } @@ -218,7 +219,7 @@ public void execute(ExecutionUnit operation, long mark_ID, boolean clean) { public void PROCESS(Context context, long mark_ID) { int threadId = context.thisThreadId; MeasureTools.BEGIN_SCHEDULE_NEXT_TIME_MEASURE(context.thisThreadId); - SchedulingUnit next = next(context); + OperationChain next = next(context); MeasureTools.END_SCHEDULE_NEXT_TIME_MEASURE(threadId); if (next != null) { @@ -251,13 +252,13 @@ public void PROCESS(Context context, long mark_ID) { * @param context * @return */ - protected SchedulingUnit next(Context context) { + protected OperationChain next(Context context) { throw new UnsupportedOperationException(); } - public boolean executeWithBusyWait(Context context, SchedulingUnit operationChain, long mark_ID) { - MyList operation_chain_list = operationChain.getOperations(); - for (ExecutionUnit operation : operation_chain_list) { + public boolean executeWithBusyWait(Context context, OperationChain operationChain, long mark_ID) { + MyList operation_chain_list = operationChain.getOperations(); + for (Operation operation : operation_chain_list) { if (operation.getOperationState().equals(MetaTypes.OperationStateType.EXECUTED) || operation.getOperationState().equals(MetaTypes.OperationStateType.ABORTED) || operation.isFailed) continue; @@ -276,19 +277,19 @@ public boolean executeWithBusyWait(Context context, SchedulingUnit operationChai return true; } - protected void checkTransactionAbort(ExecutionUnit operation, SchedulingUnit operationChain) { + protected void checkTransactionAbort(Operation operation, OperationChain operationChain) { // in coarse-grained algorithms, we will not handle transaction abort gracefully, just update the state of the operation operation.stateTransition(MetaTypes.OperationStateType.ABORTED); // save the abort information and redo the batch. } - protected SchedulingUnit nextFromBusyWaitQueue(Context context) { + protected OperationChain nextFromBusyWaitQueue(Context context) { return context.busyWaitQueue.poll(); } - protected abstract void DISTRIBUTE(SchedulingUnit task, Context context); + protected abstract void DISTRIBUTE(OperationChain task, Context context); - protected abstract void NOTIFY(SchedulingUnit task, Context context); + protected abstract void NOTIFY(OperationChain task, Context context); @Override public boolean FINISHED(Context context) { @@ -321,7 +322,48 @@ public void TxnSubmitBegin(Context context) { } @Override - public abstract void TxnSubmitFinished(Context context); + public void TxnSubmitFinished(Context context) { + MeasureTools.BEGIN_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); + // the data structure to store all operations created from the txn, store them in order, which indicates the logical dependency + List operationGraph = new ArrayList<>(); + int txnOpId = 0; + Operation headerOperation = null; + Operation set_op; + for (Request request : context.requests) { + set_op = constructOp(operationGraph, request); + if (txnOpId == 0) + headerOperation = set_op; + // addOperation an operation id for the operation for the purpose of temporal dependency construction + set_op.setTxnOpId(txnOpId++); + set_op.addHeader(headerOperation); + headerOperation.addDescendant(set_op); + } + // set logical dependencies among all operation in the same transaction + MeasureTools.END_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); + } + + private Operation constructOp(List operationGraph, Request request) { + long bid = request.txn_context.getBID(); + Operation set_op; + Context targetContext = getTargetContext(request.src_key); + switch (request.accessType) { + case READ_WRITE_COND: // they can use the same method for processing + case READ_WRITE: + set_op = new Operation(request.src_key, request.function, request.table_name, null, request.condition_records, request.condition, + request.success, request.txn_context, request.accessType, request.d_record, request.d_record, bid, targetContext); + break; + case READ_WRITE_COND_READ: + case READ_WRITE_COND_READN: + set_op = new Operation(request.src_key, request.function, request.table_name, request.record_ref, request.condition_records, request.condition, + request.success, request.txn_context, request.accessType, request.d_record, request.d_record, bid, targetContext); + break; + default: + throw new RuntimeException("Unexpected operation"); + } + operationGraph.add(set_op); + tpg.setupOperationTDFD(set_op, request, targetContext); + return set_op; + } @Override public void AddContext(int threadId, Context context) { @@ -332,9 +374,9 @@ public void AddContext(int threadId, Context context) { } } - protected boolean isConflicted(Context context, SchedulingUnit operationChain, ExecutionUnit operation) { + protected boolean isConflicted(Context context, OperationChain operationChain, Operation operation) { if (operation.fd_parents != null) { - for (AbstractOperation conditioned_operation : operation.fd_parents) { + for (Operation conditioned_operation : operation.fd_parents) { if (conditioned_operation != null) { if (!(conditioned_operation.getOperationState().equals(MetaTypes.OperationStateType.EXECUTED) || conditioned_operation.getOperationState().equals(MetaTypes.OperationStateType.ABORTED) diff --git a/state-engine/src/main/java/scheduler/impl/og/nonstructured/AbstractOGNSScheduler.java b/state-engine/src/main/java/scheduler/impl/og/nonstructured/AbstractOGNSScheduler.java index 216245dea..03e4d141e 100644 --- a/state-engine/src/main/java/scheduler/impl/og/nonstructured/AbstractOGNSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/nonstructured/AbstractOGNSScheduler.java @@ -2,12 +2,12 @@ import scheduler.context.og.AbstractOGNSContext; import scheduler.impl.og.OGScheduler; -import scheduler.struct.og.nonstructured.AbstractNSOperationChain; -import scheduler.struct.og.nonstructured.NSOperation; +import scheduler.struct.og.Operation; +import scheduler.struct.og.OperationChain; import transaction.impl.ordered.MyList; -public abstract class AbstractOGNSScheduler, ExecutionUnit extends NSOperation, SchedulingUnit extends AbstractNSOperationChain> - extends OGScheduler { +public abstract class AbstractOGNSScheduler + extends OGScheduler { public AbstractOGNSScheduler(int totalThreads, int NUM_ITEMS, int app) { super(totalThreads, NUM_ITEMS, app); @@ -38,12 +38,7 @@ public boolean FINISHED(Context context) { } @Override - public void TxnSubmitFinished(Context context) { - throw new UnsupportedOperationException("Unsupported."); - } - - @Override - protected void NOTIFY(SchedulingUnit task, Context context) { + protected void NOTIFY(OperationChain task, Context context) { throw new UnsupportedOperationException("Unsupported."); } @@ -54,9 +49,9 @@ protected void NOTIFY(SchedulingUnit task, Context context) { * @param mark_ID * @return */ - public boolean execute(Context context, SchedulingUnit operationChain, long mark_ID) { - MyList operation_chain_list = operationChain.getOperations(); - for (ExecutionUnit operation : operation_chain_list) { + public boolean execute(Context context, OperationChain operationChain, long mark_ID) { + MyList operation_chain_list = operationChain.getOperations(); + for (Operation operation : operation_chain_list) { // MeasureTools.BEGIN_SCHEDULE_USEFUL_TIME_MEASURE(context.thisThreadId); execute(operation, mark_ID, false); // MeasureTools.END_SCHEDULE_USEFUL_TIME_MEASURE(context.thisThreadId); @@ -71,8 +66,8 @@ public boolean execute(Context context, SchedulingUnit operationChain, long mark * @return */ @Override - protected SchedulingUnit next(Context context) { - SchedulingUnit operationChain = context.OCwithChildren.pollLast(); + protected OperationChain next(Context context) { + OperationChain operationChain = context.OCwithChildren.pollLast(); if (operationChain == null) { operationChain = context.IsolatedOC.pollLast(); } @@ -86,7 +81,7 @@ protected SchedulingUnit next(Context context) { * 3. shared: put all operations in a pool and */ @Override - public void DISTRIBUTE(SchedulingUnit task, Context context) { + public void DISTRIBUTE(OperationChain task, Context context) { if (task != null) { if (!task.hasChildren()) { context.IsolatedOC.add(task); diff --git a/state-engine/src/main/java/scheduler/impl/og/nonstructured/ExecutableTaskListener.java b/state-engine/src/main/java/scheduler/impl/og/nonstructured/ExecutableTaskListener.java index 29b81d013..3d98ecf30 100644 --- a/state-engine/src/main/java/scheduler/impl/og/nonstructured/ExecutableTaskListener.java +++ b/state-engine/src/main/java/scheduler/impl/og/nonstructured/ExecutableTaskListener.java @@ -1,27 +1,28 @@ package scheduler.impl.og.nonstructured; -import scheduler.struct.og.nonstructured.AbstractNSOperationChain; -import scheduler.struct.og.nonstructured.NSOperation; + +import scheduler.context.og.AbstractOGNSContext; +import scheduler.struct.og.OperationChain; /** * Register an operation to queue. */ -public class ExecutableTaskListener> { +public class ExecutableTaskListener { private final AbstractOGNSScheduler abstractOGNSScheduler; public ExecutableTaskListener(AbstractOGNSScheduler abstractOGNSScheduler) { this.abstractOGNSScheduler = abstractOGNSScheduler; } - public void onOCExecutable(SchedulingUnit operationChain) { - abstractOGNSScheduler.DISTRIBUTE(operationChain, operationChain.context);//TODO: make it clear.. + public void onOCExecutable(OperationChain operationChain) { + abstractOGNSScheduler.DISTRIBUTE(operationChain, (AbstractOGNSContext) operationChain.context);//TODO: make it clear.. } - public void onOCFinalized(SchedulingUnit operationChain) { + public void onOCFinalized(OperationChain operationChain) { operationChain.context.scheduledOPs += operationChain.getOperations().size(); } - public void onOCRollbacked(SchedulingUnit operationChain) { + public void onOCRollbacked(OperationChain operationChain) { operationChain.context.scheduledOPs += operationChain.getOperations().size(); } } diff --git a/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSAScheduler.java b/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSAScheduler.java index 6ca7876e1..5fd94af8e 100644 --- a/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSAScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSAScheduler.java @@ -1,18 +1,13 @@ package scheduler.impl.og.nonstructured; -import profiler.MeasureTools; -import scheduler.Request; import scheduler.context.og.OGNSAContext; +import scheduler.struct.og.Operation; +import scheduler.struct.og.OperationChain; import scheduler.struct.op.MetaTypes; -import scheduler.struct.og.nonstructured.NSAOperationChain; -import scheduler.struct.og.nonstructured.NSAOperation; import transaction.impl.ordered.MyList; import utils.SOURCE_CONTROL; -import java.util.ArrayList; -import java.util.List; - -public class OGNSAScheduler extends AbstractOGNSScheduler { +public class OGNSAScheduler extends AbstractOGNSScheduler { public final ExecutableTaskListener executableTaskListener = new ExecutableTaskListener(); @@ -43,58 +38,7 @@ public void EXPLORE(OGNSAContext context) { } @Override - public void TxnSubmitFinished(OGNSAContext context) { - MeasureTools.BEGIN_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); - // the data structure to store all operations created from the txn, store them in order, which indicates the logical dependency - List operationGraph = new ArrayList<>(); - int txnOpId = 0; - NSAOperation headerOperation = null; - NSAOperation set_op; - for (Request request : context.requests) { - set_op = constructOp(operationGraph, request); - if (txnOpId == 0) - headerOperation = set_op; - // addOperation an operation id for the operation for the purpose of temporal dependency construction - set_op.setTxnOpId(txnOpId++); - set_op.addHeader(headerOperation); - headerOperation.addDescendant(set_op); - } - // set logical dependencies among all operation in the same transaction - MeasureTools.END_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); - } - - - private NSAOperation constructOp(List operationGraph, Request request) { - long bid = request.txn_context.getBID(); - NSAOperation set_op; - OGNSAContext targetContext = getTargetContext(request.src_key); - switch (request.accessType) { - case READ_WRITE_COND: // they can use the same method for processing - case READ_WRITE: - set_op = new NSAOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.function, request.condition, request.condition_records, request.success); - break; - case READ_WRITE_COND_READ: - case READ_WRITE_COND_READN: - set_op = new NSAOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); - break; - case READ_WRITE_READ: - set_op=new NSAOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function); - break; - default: - throw new RuntimeException("Unexpected operation"); - } - operationGraph.add(set_op); -// set_op.setConditionSources(request.condition_sourceTable, request.condition_source); -// tpg.cacheToSortedOperations(set_op); - tpg.setupOperationTDFD(set_op, request, targetContext); - return set_op; - } - - @Override - protected void NOTIFY(NSAOperationChain task, OGNSAContext context) { + protected void NOTIFY(OperationChain task, OGNSAContext context) { context.partitionStateManager.onOcExecuted(task); } @@ -106,9 +50,9 @@ protected void NOTIFY(NSAOperationChain task, OGNSAContext context) { * @return */ @Override - public boolean execute(OGNSAContext context, NSAOperationChain operationChain, long mark_ID) { - MyList operation_chain_list = operationChain.getOperations(); - for (NSAOperation operation : operation_chain_list) { + public boolean execute(OGNSAContext context, OperationChain operationChain, long mark_ID) { + MyList operation_chain_list = operationChain.getOperations(); + for (Operation operation : operation_chain_list) { // MeasureTools.BEGIN_SCHEDULE_USEFUL_TIME_MEASURE(context.thisThreadId); execute(operation, mark_ID, false); checkTransactionAbort(operation, operationChain); @@ -118,7 +62,7 @@ public boolean execute(OGNSAContext context, NSAOperationChain operationChain, l } @Override - protected void checkTransactionAbort(NSAOperation operation, NSAOperationChain operationChain) { + protected void checkTransactionAbort(Operation operation, OperationChain operationChain) { if (operation.isFailed && !operation.getOperationState().equals(MetaTypes.OperationStateType.ABORTED)) { operationChain.needAbortHandling = true; operationChain.failedOperations.add(operation); @@ -129,15 +73,15 @@ protected void checkTransactionAbort(NSAOperation operation, NSAOperationChain o * Register an operation to queue. */ public class ExecutableTaskListener { - public void onOCExecutable(NSAOperationChain operationChain) { + public void onOCExecutable(OperationChain operationChain) { DISTRIBUTE(operationChain, (OGNSAContext) operationChain.context); } - public void onOCFinalized(NSAOperationChain operationChain) { + public void onOCFinalized(OperationChain operationChain) { operationChain.context.scheduledOPs += operationChain.getOperations().size(); } - public void onOCRollbacked(NSAOperationChain operationChain) { + public void onOCRollbacked(OperationChain operationChain) { operationChain.context.scheduledOPs -= operationChain.getOperations().size(); } } diff --git a/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSScheduler.java b/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSScheduler.java index 404034df5..18adc8a81 100644 --- a/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/nonstructured/OGNSScheduler.java @@ -2,20 +2,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import profiler.MeasureTools; -import scheduler.Request; import scheduler.context.og.OGNSContext; +import scheduler.struct.og.Operation; +import scheduler.struct.og.OperationChain; import scheduler.struct.op.MetaTypes; -import scheduler.struct.og.nonstructured.NSOperation; -import scheduler.struct.og.nonstructured.NSOperationChain; import utils.SOURCE_CONTROL; -import java.util.ArrayList; -import java.util.List; - import static common.CONTROL.enable_log; -public class OGNSScheduler extends AbstractOGNSScheduler { +public class OGNSScheduler extends AbstractOGNSScheduler { private static final Logger log = LoggerFactory.getLogger(OGNSScheduler.class); public ExecutableTaskListener executableTaskListener = new ExecutableTaskListener(); @@ -70,12 +65,12 @@ public void EXPLORE(OGNSContext context) { } @Override - protected void NOTIFY(NSOperationChain task, OGNSContext context) { + protected void NOTIFY(OperationChain task, OGNSContext context) { context.partitionStateManager.onOcExecuted(task); } @Override - protected void checkTransactionAbort(NSOperation operation, NSOperationChain operationChain) { + protected void checkTransactionAbort(Operation operation, OperationChain operationChain) { // in coarse-grained algorithms, we will not handle transaction abort gracefully, just update the state of the operation operation.stateTransition(MetaTypes.OperationStateType.ABORTED); // save the abort information and redo the batch. @@ -83,57 +78,19 @@ protected void checkTransactionAbort(NSOperation operation, NSOperationChain ope } - @Override - public void TxnSubmitFinished(OGNSContext context) { - MeasureTools.BEGIN_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); - // the data structure to store all operations created from the txn, store them in order, which indicates the logical dependency - List operationGraph = new ArrayList<>(); - for (Request request : context.requests) { - constructOp(operationGraph, request, context); - } - // set logical dependencies among all operation in the same transaction - MeasureTools.END_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); - } - - private void constructOp(List operationGraph, Request request, OGNSContext context) { - long bid = request.txn_context.getBID(); - NSOperation set_op; - OGNSContext targetContext = getTargetContext(request.src_key); - switch (request.accessType) { - case READ_WRITE_COND: // they can use the same method for processing - case READ_WRITE: - set_op = new NSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.function, request.condition, request.condition_records, request.success); - break; - case READ_WRITE_COND_READ: - case READ_WRITE_COND_READN: - set_op = new NSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); - break; - case READ_WRITE_READ: - set_op = new NSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function); - break; - default: - throw new RuntimeException("Unexpected operation"); - } - operationGraph.add(set_op); - tpg.setupOperationTDFD(set_op, request, targetContext); - } - /** * Register an operation to queue. */ public class ExecutableTaskListener { - public void onOCExecutable(NSOperationChain operationChain) { + public void onOCExecutable(OperationChain operationChain) { DISTRIBUTE(operationChain, (OGNSContext) operationChain.context);//TODO: make it clear.. } - public void onOCFinalized(NSOperationChain operationChain) { + public void onOCFinalized(OperationChain operationChain) { operationChain.context.scheduledOPs += operationChain.getOperations().size(); } - public void onOCRollbacked(NSOperationChain operationChain) { + public void onOCRollbacked(OperationChain operationChain) { operationChain.context.scheduledOPs -= operationChain.getOperations().size(); } } diff --git a/state-engine/src/main/java/scheduler/impl/og/structured/AbstractOGBFSScheduler.java b/state-engine/src/main/java/scheduler/impl/og/structured/AbstractOGBFSScheduler.java index 7c4ac268b..5ee09172a 100644 --- a/state-engine/src/main/java/scheduler/impl/og/structured/AbstractOGBFSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/structured/AbstractOGBFSScheduler.java @@ -1,8 +1,7 @@ package scheduler.impl.og.structured; -import scheduler.context.og.OGBFSContext; -import scheduler.struct.og.structured.bfs.BFSOperation; -import scheduler.struct.og.structured.bfs.BFSOperationChain; +import scheduler.context.og.OGSContext; +import scheduler.struct.og.OperationChain; import utils.SOURCE_CONTROL; /** @@ -12,7 +11,7 @@ * 3. thread will find operations from its queue for execution. * It's a shared data structure! */ -public class AbstractOGBFSScheduler extends OGSScheduler { +public class AbstractOGBFSScheduler extends OGSScheduler { public AbstractOGBFSScheduler(int totalThreads, int NUM_ITEMS, int app) { super(totalThreads, NUM_ITEMS, app); @@ -22,7 +21,7 @@ public AbstractOGBFSScheduler(int totalThreads, int NUM_ITEMS, int app) { @Override public void EXPLORE(Context context) { - BFSOperationChain next = Next(context); + OperationChain next = Next(context); if (next == null && !context.finished()) { //current level is all processed at the current thread. while (next == null) { SOURCE_CONTROL.getInstance().waitForOtherThreads(); @@ -34,10 +33,6 @@ public void EXPLORE(Context context) { } @Override - protected void NOTIFY(BFSOperationChain operationChain, Context context) { - } - - @Override - public void TxnSubmitFinished(Context context) { + protected void NOTIFY(OperationChain operationChain, Context context) { } } \ No newline at end of file diff --git a/state-engine/src/main/java/scheduler/impl/og/structured/AbstractOGDFSScheduler.java b/state-engine/src/main/java/scheduler/impl/og/structured/AbstractOGDFSScheduler.java index ebf056ff9..12bfd52bf 100644 --- a/state-engine/src/main/java/scheduler/impl/og/structured/AbstractOGDFSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/structured/AbstractOGDFSScheduler.java @@ -1,8 +1,7 @@ package scheduler.impl.og.structured; -import scheduler.context.og.OGDFSContext; -import scheduler.struct.og.structured.dfs.DFSOperation; -import scheduler.struct.og.structured.dfs.DFSOperationChain; +import scheduler.context.og.OGSContext; +import scheduler.struct.og.OperationChain; import java.util.Collection; @@ -13,7 +12,7 @@ * 3. thread will find operations from its queue for execution. * It's a shared data structure! */ -public class AbstractOGDFSScheduler extends OGSScheduler { +public class AbstractOGDFSScheduler extends OGSScheduler { public AbstractOGDFSScheduler(int totalThreads, int NUM_ITEMS, int app) { @@ -22,7 +21,7 @@ public AbstractOGDFSScheduler(int totalThreads, int NUM_ITEMS, int app) { @Override public void EXPLORE(Context context) { - DFSOperationChain oc = Next(context); + OperationChain oc = Next(context); while (oc == null) { if (context.exploreFinished()) break; @@ -40,16 +39,12 @@ public void EXPLORE(Context context) { * @param context */ @Override - protected void NOTIFY(DFSOperationChain operationChain, Context context) { + protected void NOTIFY(OperationChain operationChain, Context context) { // context.partitionStateManager.onOcExecuted(operationChain); operationChain.isExecuted = true; // set operation chain to be executed, which is used for further rollback - Collection ocs = operationChain.getChildren(); - for (DFSOperationChain childOC : ocs) { + Collection ocs = operationChain.getChildren(); + for (OperationChain childOC : ocs) { childOC.updateDependency(); } } - - @Override - public void TxnSubmitFinished(Context context) { - } } diff --git a/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSAScheduler.java b/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSAScheduler.java index b287eecdb..2c13490a0 100644 --- a/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSAScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSAScheduler.java @@ -4,10 +4,10 @@ import org.slf4j.LoggerFactory; import profiler.MeasureTools; import scheduler.Request; -import scheduler.context.og.OGBFSAContext; +import scheduler.context.og.OGSAContext; +import scheduler.struct.og.Operation; +import scheduler.struct.og.OperationChain; import scheduler.struct.op.MetaTypes; -import scheduler.struct.og.structured.bfs.BFSOperation; -import scheduler.struct.og.structured.bfs.BFSOperationChain; import utils.SOURCE_CONTROL; import java.util.ArrayList; @@ -26,9 +26,9 @@ * 3. thread will find operations from its queue for execution. * It's a shared data structure! */ -public class OGBFSAScheduler extends AbstractOGBFSScheduler { +public class OGBFSAScheduler extends AbstractOGBFSScheduler { private static final Logger LOG = LoggerFactory.getLogger(OGBFSAScheduler.class); - public final ConcurrentLinkedDeque failedOperations = new ConcurrentLinkedDeque<>();//aborted operations per thread. + public final ConcurrentLinkedDeque failedOperations = new ConcurrentLinkedDeque<>();//aborted operations per thread. public final AtomicBoolean needAbortHandling = new AtomicBoolean(false);//if any operation is aborted during processing. public int targetRollbackLevel = 0;//shared data structure. @@ -37,8 +37,8 @@ public OGBFSAScheduler(int totalThreads, int NUM_ITEMS, int app) { } @Override - public void EXPLORE(OGBFSAContext context) { - BFSOperationChain next = Next(context); + public void EXPLORE(OGSAContext context) { + OperationChain next = Next(context); if (next == null && !context.exploreFinished()) { //current level is all processed at the current thread. while (next == null) { SOURCE_CONTROL.getInstance().waitForOtherThreads(); @@ -71,8 +71,8 @@ public void EXPLORE(OGBFSAContext context) { // * @param mark_ID // */ // @Override -// public void execute(OGBFSAContext context, MyList operation_chain, long mark_ID) { -// for (BFSOperation operation : operation_chain) { +// public void execute(OGSAContext context, MyList operation_chain, long mark_ID) { +// for (Operation operation : operation_chain) { //// MeasureTools.BEGIN_SCHEDULE_USEFUL_TIME_MEASURE(context.thisThreadId); // execute(operation, mark_ID, false); // checkTransactionAbort(operation, operation_chain); @@ -88,9 +88,9 @@ public void EXPLORE(OGBFSAContext context) { // * @return // */ // @Override -// public boolean executeWithBusyWait(OGBFSAContext context, BFSOperationChain operationChain, long mark_ID) { -// MyList operation_chain_list = operationChain.getOperations(); -// for (BFSOperation operation : operation_chain_list) { +// public boolean executeWithBusyWait(OGSAContext context, OperationChain operationChain, long mark_ID) { +// MyList operation_chain_list = operationChain.getOperations(); +// for (Operation operation : operation_chain_list) { //// MeasureTools.BEGIN_SCHEDULE_USEFUL_TIME_MEASURE(context.thisThreadId); // if (operation.getOperationState().equals(MetaTypes.OperationStateType.EXECUTED)) continue; // if (isConflicted(context, operationChain, operation)) return false; // did not completed @@ -105,50 +105,11 @@ public void EXPLORE(OGBFSAContext context) { // } @Override - protected void NOTIFY(BFSOperationChain operationChain, OGBFSAContext context) { + protected void NOTIFY(OperationChain operationChain, OGSAContext context) { } @Override - public void TxnSubmitFinished(OGBFSAContext context) { - MeasureTools.BEGIN_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); - // the data structure to store all operations created from the txn, store them in order, which indicates the logical dependency - List operationGraph = new ArrayList<>(); - for (Request request : context.requests) { - constructOp(operationGraph, request); - } - MeasureTools.END_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); - } - - private void constructOp(List operationGraph, Request request) { - long bid = request.txn_context.getBID(); - BFSOperation set_op; - OGBFSAContext targetContext = getTargetContext(request.src_key); - switch (request.accessType) { - case READ_WRITE_COND: // they can use the same method for processing - case READ_WRITE: - set_op = new BFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.function, request.condition, request.condition_records, request.success); - break; - case READ_WRITE_COND_READ: - case READ_WRITE_COND_READN: - set_op = new BFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); - break; - case READ_WRITE_READ: - set_op = new BFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function); - break; - default: - throw new UnsupportedOperationException(); - } - operationGraph.add(set_op); -// set_op.setConditionSources(request.condition_sourceTable, request.condition_source); -// tpg.cacheToSortedOperations(set_op); - tpg.setupOperationTDFD(set_op, request, targetContext); - } - - @Override - protected void checkTransactionAbort(BFSOperation operation, BFSOperationChain operationChain) { + protected void checkTransactionAbort(Operation operation, OperationChain operationChain) { if (operation.isFailed && !operation.getOperationState().equals(MetaTypes.OperationStateType.ABORTED)) { needAbortHandling.compareAndSet(false, true); @@ -156,7 +117,7 @@ protected void checkTransactionAbort(BFSOperation operation, BFSOperationChain o } } - protected void abortHandling(OGBFSAContext context) { + protected void abortHandling(OGSAContext context) { MarkOperationsToAbort(context); SOURCE_CONTROL.getInstance().waitForOtherThreads(); @@ -169,15 +130,15 @@ protected void abortHandling(OGBFSAContext context) { } //TODO: mark operations of aborted transaction to be aborted. - protected void MarkOperationsToAbort(OGBFSAContext context) { + protected void MarkOperationsToAbort(OGSAContext context) { boolean markAny = false; - ArrayList operationChains; + ArrayList operationChains; int curLevel; - for (Map.Entry> operationChainsEntry : context.allocatedLayeredOCBucket.entrySet()) { + for (Map.Entry> operationChainsEntry : context.allocatedLayeredOCBucket.entrySet()) { operationChains = operationChainsEntry.getValue(); curLevel = operationChainsEntry.getKey(); - for (BFSOperationChain operationChain : operationChains) { - for (BFSOperation operation : operationChain.getOperations()) { + for (OperationChain operationChain : operationChains) { + for (Operation operation : operationChain.getOperations()) { markAny |= _MarkOperationsToAbort(context, operation); } } @@ -199,11 +160,11 @@ protected void MarkOperationsToAbort(OGBFSAContext context) { * @param operation * @return */ - private boolean _MarkOperationsToAbort(OGBFSAContext context, BFSOperation operation) { + private boolean _MarkOperationsToAbort(OGSAContext context, Operation operation) { long bid = operation.bid; boolean markAny = false; //identify bids to be aborted. - for (BFSOperation failedOp : failedOperations) { + for (Operation failedOp : failedOperations) { if (bid == failedOp.bid) { // operation.aborted = true; operation.stateTransition(MetaTypes.OperationStateType.ABORTED); @@ -213,7 +174,7 @@ private boolean _MarkOperationsToAbort(OGBFSAContext context, BFSOperation opera return markAny; } - protected void IdentifyRollbackLevel(OGBFSAContext context) { + protected void IdentifyRollbackLevel(OGSAContext context) { if (context.thisThreadId == 0) { targetRollbackLevel = Integer.MAX_VALUE; for (int i = 0; i < tpg.totalThreads; i++) { // find the first level that contains aborted operations @@ -222,12 +183,12 @@ protected void IdentifyRollbackLevel(OGBFSAContext context) { } } - protected void SetRollbackLevel(OGBFSAContext context) { + protected void SetRollbackLevel(OGSAContext context) { if (enable_log) LOG.debug("++++++ rollback at: " + targetRollbackLevel); context.rollbackLevel = targetRollbackLevel; } - protected void ResumeExecution(OGBFSAContext context) { + protected void ResumeExecution(OGSAContext context) { context.rollbackLevel = -1; context.isRollbacked = false; // if (context.thisThreadId == 0) { // TODO: what should we do to optimize this part? @@ -238,7 +199,7 @@ protected void ResumeExecution(OGBFSAContext context) { if (enable_log) LOG.debug("+++++++ rollback completed..."); } - protected void RollbackToCorrectLayerForRedo(OGBFSAContext context) { + protected void RollbackToCorrectLayerForRedo(OGSAContext context) { int level; for (level = context.rollbackLevel; level <= context.currentLevel; level++) { context.scheduledOPs -= getNumOPsByLevel(context, level); @@ -248,10 +209,10 @@ protected void RollbackToCorrectLayerForRedo(OGBFSAContext context) { context.currentLevel = context.rollbackLevel - 1; } - protected int getNumOPsByLevel(OGBFSAContext context, int level) { + protected int getNumOPsByLevel(OGSAContext context, int level) { int ops = 0; if (context.allocatedLayeredOCBucket.containsKey(level)) { // oc level may not be sequential - for (BFSOperationChain operationChain : context.allocatedLayeredOCBucket.get(level)) { + for (OperationChain operationChain : context.allocatedLayeredOCBucket.get(level)) { ops += operationChain.getOperations().size(); } } diff --git a/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSScheduler.java b/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSScheduler.java index f604a2780..e1a4198e0 100644 --- a/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/structured/OGBFSScheduler.java @@ -1,15 +1,9 @@ package scheduler.impl.og.structured; -import profiler.MeasureTools; -import scheduler.Request; -import scheduler.context.og.OGBFSContext; -import scheduler.struct.og.structured.bfs.BFSOperation; -import scheduler.struct.og.structured.bfs.BFSOperationChain; +import scheduler.context.og.OGSContext; +import scheduler.struct.og.OperationChain; import utils.SOURCE_CONTROL; -import java.util.ArrayList; -import java.util.List; - /** * The scheduler based on TPG, this is to be invoked when the queue is empty of each thread, it works as follows: * 1. explore dependencies in TPG, and find out the ready/speculative operations @@ -17,15 +11,15 @@ * 3. thread will find operations from its queue for execution. * It's a shared data structure! */ -public class OGBFSScheduler extends AbstractOGBFSScheduler { +public class OGBFSScheduler extends AbstractOGBFSScheduler { public OGBFSScheduler(int totalThreads, int NUM_ITEMS, int app) { super(totalThreads, NUM_ITEMS, app); } @Override - public void EXPLORE(OGBFSContext context) { - BFSOperationChain next = Next(context); + public void EXPLORE(OGSContext context) { + OperationChain next = Next(context); if (next == null && !context.exploreFinished()) { //current level is all processed at the current thread. while (next == null) { SOURCE_CONTROL.getInstance().waitForOtherThreads(); @@ -36,50 +30,6 @@ public void EXPLORE(OGBFSContext context) { DISTRIBUTE(next, context); } - @Override - protected void NOTIFY(BFSOperationChain operationChain, OGBFSContext context) { - } - - @Override - public void TxnSubmitFinished(OGBFSContext context) { - MeasureTools.BEGIN_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); - // the data structure to store all operations created from the txn, store them in order, which indicates the logical dependency - List operationGraph = new ArrayList<>(); - for (Request request : context.requests) { - constructOp(operationGraph, request); - } - MeasureTools.END_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); - } - - private void constructOp(List operationGraph, Request request) { - long bid = request.txn_context.getBID(); - BFSOperation set_op = null; - OGBFSContext targetContext = getTargetContext(request.src_key); - switch (request.accessType) { - case READ_WRITE_COND: // they can use the same method for processing - case READ_WRITE: - set_op = new BFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.function, request.condition, request.condition_records, request.success); - break; - case READ_WRITE_COND_READ: - case READ_WRITE_COND_READN: - set_op = new BFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); - break; - case READ_WRITE_READ: - set_op = new BFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function); - break; - default: - throw new UnsupportedOperationException(); - } - operationGraph.add(set_op); -// set_op.setConditionSources(request.condition_sourceTable, request.condition_source); -// tpg.cacheToSortedOperations(set_op); -// tpg.setupOperationTDFD(set_op); - tpg.setupOperationTDFD(set_op, request, targetContext); - } - // /** // * Used by OGNSScheduler. // * @param context @@ -88,9 +38,9 @@ private void constructOp(List operationGraph, Request request) { // * @return // */ // @Override -// public boolean executeWithBusyWait(OGBFSContext context, BFSOperationChain operationChain, long mark_ID) { -// MyList operation_chain_list = operationChain.getOperations(); -// for (BFSOperation operation : operation_chain_list) { +// public boolean executeWithBusyWait(OGSContext context, OperationChain operationChain, long mark_ID) { +// MyList operation_chain_list = operationChain.getOperations(); +// for (Operation operation : operation_chain_list) { // if (operation.getOperationState().equals(MetaTypes.OperationStateType.EXECUTED)) continue; // if (isConflicted(context, operationChain, operation)) return false; // did not completed // execute(operation, mark_ID, false); diff --git a/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSAScheduler.java b/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSAScheduler.java index fbb6351bc..a294b68cd 100644 --- a/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSAScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSAScheduler.java @@ -2,17 +2,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import profiler.MeasureTools; -import scheduler.Request; -import scheduler.context.og.OGDFSAContext; +import scheduler.context.og.OGSAContext; +import scheduler.struct.og.Operation; +import scheduler.struct.og.OperationChain; import scheduler.struct.op.MetaTypes; -import scheduler.struct.og.structured.dfs.DFSOperation; -import scheduler.struct.og.structured.dfs.DFSOperationChain; import utils.SOURCE_CONTROL; import java.util.ArrayList; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.atomic.AtomicBoolean; @@ -26,10 +23,10 @@ * 3. thread will find operations from its queue for execution. * It's a shared data structure! */ -public class OGDFSAScheduler extends AbstractOGDFSScheduler { +public class OGDFSAScheduler extends AbstractOGDFSScheduler { private static final Logger LOG = LoggerFactory.getLogger(OGDFSAScheduler.class); - public final ConcurrentLinkedDeque failedOperations = new ConcurrentLinkedDeque<>();//aborted operations per thread. + public final ConcurrentLinkedDeque failedOperations = new ConcurrentLinkedDeque<>();//aborted operations per thread. public final AtomicBoolean needAbortHandling = new AtomicBoolean(false);//if any operation is aborted during processing. public OGDFSAScheduler(int totalThreads, int NUM_ITEMS, int app) { @@ -37,8 +34,8 @@ public OGDFSAScheduler(int totalThreads, int NUM_ITEMS, int app) { } @Override - public void EXPLORE(OGDFSAContext context) { - DFSOperationChain oc = Next(context); + public void EXPLORE(OGSAContext context) { + OperationChain oc = Next(context); while (oc == null) { // current thread finishes the current level if (needAbortHandling.get()) { @@ -66,8 +63,8 @@ public void EXPLORE(OGDFSAContext context) { // * @param mark_ID // */ // @Override -// public void execute(OGDFSAContext context, MyList operation_chain, long mark_ID) { -// for (DFSOperation operation : operation_chain) { +// public void execute(OGSAContext context, MyList operation_chain, long mark_ID) { +// for (Operation operation : operation_chain) { //// MeasureTools.BEGIN_SCHEDULE_USEFUL_TIME_MEASURE(context.thisThreadId); // execute(operation, mark_ID, false); // checkTransactionAbort(operation); @@ -83,9 +80,9 @@ public void EXPLORE(OGDFSAContext context) { // * @return // */ // @Override -// public boolean executeWithBusyWait(OGDFSAContext context, DFSOperationChain operationChain, long mark_ID) { -// MyList operation_chain_list = operationChain.getOperations(); -// for (DFSOperation operation : operation_chain_list) { +// public boolean executeWithBusyWait(OGSAContext context, OperationChain operationChain, long mark_ID) { +// MyList operation_chain_list = operationChain.getOperations(); +// for (Operation operation : operation_chain_list) { //// MeasureTools.BEGIN_SCHEDULE_USEFUL_TIME_MEASURE(context.thisThreadId); // if (operation.isExecuted) continue; // if (isConflicted(context, operationChain, operation)) return false; // did not completed @@ -103,58 +100,19 @@ public void EXPLORE(OGDFSAContext context) { * @param context */ @Override - protected void NOTIFY(DFSOperationChain operationChain, OGDFSAContext context) { + protected void NOTIFY(OperationChain operationChain, OGSAContext context) { // context.partitionStateManager.onOcExecuted(operationChain); operationChain.isExecuted = true; // set operation chain to be executed, which is used for further rollback - Collection ocs = operationChain.getChildren(); - for (DFSOperationChain childOC : ocs) { + Collection ocs = operationChain.getChildren(); + for (OperationChain childOC : ocs) { childOC.updateDependency(); } } @Override - public void TxnSubmitFinished(OGDFSAContext context) { - MeasureTools.BEGIN_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); - // the data structure to store all operations created from the txn, store them in order, which indicates the logical dependency - List operationGraph = new ArrayList<>(); - for (Request request : context.requests) { - constructOp(operationGraph, request); - } - MeasureTools.END_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); - } - - private void constructOp(List operationGraph, Request request) { - long bid = request.txn_context.getBID(); - DFSOperation set_op = null; - OGDFSAContext targetContext = getTargetContext(request.src_key); - switch (request.accessType) { - case READ_WRITE_COND: // they can use the same method for processing - case READ_WRITE: - set_op = new DFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.function, request.condition, request.condition_records, request.success); - break; - case READ_WRITE_COND_READ: - case READ_WRITE_COND_READN: - set_op = new DFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); - break; - case READ_WRITE_READ: - set_op = new DFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function); - break; - default: - throw new UnsupportedOperationException(); - } - operationGraph.add(set_op); -// set_op.setConditionSources(request.condition_sourceTable, request.condition_source); -// tpg.cacheToSortedOperations(set_op); - tpg.setupOperationTDFD(set_op, request, targetContext); - } - - @Override - protected void checkTransactionAbort(DFSOperation operation, DFSOperationChain dfsOperationChain) { + protected void checkTransactionAbort(Operation operation, OperationChain OperationChain) { if (operation.isFailed && !operation.getOperationState().equals(MetaTypes.OperationStateType.ABORTED)) { - for (DFSOperation failedOp : failedOperations) { + for (Operation failedOp : failedOperations) { if (failedOp.bid == operation.bid) { return; } @@ -164,21 +122,21 @@ protected void checkTransactionAbort(DFSOperation operation, DFSOperationChain d } } - protected void abortHandling(OGDFSAContext context) { + protected void abortHandling(OGSAContext context) { MarkOperationsToAbort(context); RollbackToCorrectLayerForRedo(context); ResumeExecution(context); } - protected void MarkOperationsToAbort(OGDFSAContext context) { + protected void MarkOperationsToAbort(OGSAContext context) { boolean markAny = false; - ArrayList operationChains; + ArrayList operationChains; int curLevel; - for (Map.Entry> operationChainsEntry : context.allocatedLayeredOCBucket.entrySet()) { + for (Map.Entry> operationChainsEntry : context.allocatedLayeredOCBucket.entrySet()) { operationChains = operationChainsEntry.getValue(); curLevel = operationChainsEntry.getKey(); - for (DFSOperationChain operationChain : operationChains) { - for (DFSOperation operation : operationChain.getOperations()) { + for (OperationChain operationChain : operationChains) { + for (Operation operation : operationChain.getOperations()) { markAny |= _MarkOperationsToAbort(context, operation); } } @@ -200,11 +158,11 @@ protected void MarkOperationsToAbort(OGDFSAContext context) { * @param operation * @return */ - private boolean _MarkOperationsToAbort(OGDFSAContext context, DFSOperation operation) { + private boolean _MarkOperationsToAbort(OGSAContext context, Operation operation) { long bid = operation.bid; boolean markAny = false; //identify bids to be aborted. - for (DFSOperation failedOp : failedOperations) { + for (Operation failedOp : failedOperations) { if (bid == failedOp.bid) { operation.stateTransition(MetaTypes.OperationStateType.ABORTED); markAny = true; @@ -213,7 +171,7 @@ private boolean _MarkOperationsToAbort(OGDFSAContext context, DFSOperation opera return markAny; } - protected void ResumeExecution(OGDFSAContext context) { + protected void ResumeExecution(OGSAContext context) { context.rollbackLevel = -1; context.isRollbacked = false; @@ -223,7 +181,7 @@ protected void ResumeExecution(OGDFSAContext context) { LOG.debug("resume: " + context.thisThreadId); } - protected void RollbackToCorrectLayerForRedo(OGDFSAContext context) { + protected void RollbackToCorrectLayerForRedo(OGSAContext context) { int level; for (level = context.rollbackLevel; level <= context.currentLevel; level++) { context.scheduledOPs -= getNumOPsByLevel(context, level); @@ -233,15 +191,15 @@ protected void RollbackToCorrectLayerForRedo(OGDFSAContext context) { context.currentLevel = context.rollbackLevel - 1; } - protected int getNumOPsByLevel(OGDFSAContext context, int level) { + protected int getNumOPsByLevel(OGSAContext context, int level) { int ops = 0; if (context.allocatedLayeredOCBucket.containsKey(level)) { // oc level may not be sequential - for (DFSOperationChain operationChain : context.allocatedLayeredOCBucket.get(level)) { + for (OperationChain operationChain : context.allocatedLayeredOCBucket.get(level)) { ops += operationChain.getOperations().size(); if (operationChain.isExecuted) { // rollback children counter if the parent has been rollback operationChain.isExecuted = false; - Collection ocs = operationChain.getChildren(); - for (DFSOperationChain childOC : ocs) { + Collection ocs = operationChain.getChildren(); + for (OperationChain childOC : ocs) { childOC.rollbackDependency(); } } diff --git a/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSScheduler.java b/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSScheduler.java index ecc356cf0..f2159ba72 100644 --- a/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/structured/OGDFSScheduler.java @@ -1,14 +1,9 @@ package scheduler.impl.og.structured; -import profiler.MeasureTools; -import scheduler.Request; -import scheduler.context.og.OGDFSContext; -import scheduler.struct.og.structured.dfs.DFSOperation; -import scheduler.struct.og.structured.dfs.DFSOperationChain; +import scheduler.context.og.OGSContext; +import scheduler.struct.og.OperationChain; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; /** * The scheduler based on TPG, this is to be invoked when the queue is empty of each thread, it works as follows: @@ -17,7 +12,7 @@ * 3. thread will find operations from its queue for execution. * It's a shared data structure! */ -public class OGDFSScheduler extends AbstractOGDFSScheduler { +public class OGDFSScheduler extends AbstractOGDFSScheduler { public OGDFSScheduler(int totalThreads, int NUM_ITEMS, int app) { @@ -31,54 +26,15 @@ public OGDFSScheduler(int totalThreads, int NUM_ITEMS, int app) { * @param context */ @Override - protected void NOTIFY(DFSOperationChain operationChain, OGDFSContext context) { + protected void NOTIFY(OperationChain operationChain, OGSContext context) { // context.partitionStateManager.onOcExecuted(operationChain); operationChain.isExecuted = true; // set operation chain to be executed, which is used for further rollback - Collection ocs = operationChain.getChildren(); - for (DFSOperationChain childOC : ocs) { + Collection ocs = operationChain.getChildren(); + for (OperationChain childOC : ocs) { childOC.updateDependency(); } } - @Override - public void TxnSubmitFinished(OGDFSContext context) { - MeasureTools.BEGIN_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); - // the data structure to store all operations created from the txn, store them in order, which indicates the logical dependency - List operationGraph = new ArrayList<>(); - for (Request request : context.requests) { - constructOp(operationGraph, request); - } - MeasureTools.END_TPG_CONSTRUCTION_TIME_MEASURE(context.thisThreadId); - } - - private void constructOp(List operationGraph, Request request) { - long bid = request.txn_context.getBID(); - DFSOperation set_op = null; - OGDFSContext targetContext = getTargetContext(request.src_key); - switch (request.accessType) { - case READ_WRITE_COND: // they can use the same method for processing - case READ_WRITE: - set_op = new DFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.function, request.condition, request.condition_records, request.success); - break; - case READ_WRITE_COND_READ: - case READ_WRITE_COND_READN: - set_op = new DFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function, request.condition, request.condition_records, request.success); - break; - case READ_WRITE_READ: - set_op = new DFSOperation(request.src_key, targetContext, request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function); - break; - default: - throw new UnsupportedOperationException(); - } - operationGraph.add(set_op); -// set_op.setConditionSources(request.condition_sourceTable, request.condition_source); -// tpg.cacheToSortedOperations(set_op); - tpg.setupOperationTDFD(set_op, request, targetContext); - } - // /** // * Used by OGNSScheduler. @@ -88,9 +44,9 @@ private void constructOp(List operationGraph, Request request) { // * @return // */ // @Override -// public boolean executeWithBusyWait(OGDFSContext context, DFSOperationChain operationChain, long mark_ID) { -// MyList operation_chain_list = operationChain.getOperations(); -// for (DFSOperation operation : operation_chain_list) { +// public boolean executeWithBusyWait(OGSContext context, OperationChain operationChain, long mark_ID) { +// MyList operation_chain_list = operationChain.getOperations(); +// for (Operation operation : operation_chain_list) { // if (operation.isExecuted) continue; // if (isConflicted(context, operationChain, operation)) return false; // did not completed // execute(operation, mark_ID, false); diff --git a/state-engine/src/main/java/scheduler/impl/og/structured/OGSScheduler.java b/state-engine/src/main/java/scheduler/impl/og/structured/OGSScheduler.java index 7563f8ae9..a5f2ecec7 100644 --- a/state-engine/src/main/java/scheduler/impl/og/structured/OGSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/structured/OGSScheduler.java @@ -2,15 +2,14 @@ import scheduler.context.og.OGSContext; import scheduler.impl.og.OGScheduler; -import scheduler.struct.og.AbstractOperation; -import scheduler.struct.og.structured.SOperationChain; +import scheduler.struct.og.Operation; +import scheduler.struct.og.OperationChain; import transaction.impl.ordered.MyList; import utils.SOURCE_CONTROL; import java.util.ArrayList; -public abstract class OGSScheduler, ExecutionUnit extends AbstractOperation, SchedulingUnit extends SOperationChain> - extends OGScheduler { +public abstract class OGSScheduler extends OGScheduler { public OGSScheduler(int totalThreads, int NUM_ITEMS, int app) { super(totalThreads, NUM_ITEMS, app); @@ -34,7 +33,7 @@ protected void ProcessedToNextLevel(Context context) { // public void PROCESS(Context context, long mark_ID) { // int threadId = context.thisThreadId; // MeasureTools.BEGIN_SCHEDULE_NEXT_TIME_MEASURE(threadId); -// SchedulingUnit next = next(context); +// OperationChain next = next(context); // MeasureTools.END_SCHEDULE_NEXT_TIME_MEASURE(threadId); // if (next != null) { //// execute(context, next.getOperations(), mark_ID); @@ -62,8 +61,8 @@ protected void ProcessedToNextLevel(Context context) { * @param operation_chain * @param mark_ID */ - public void execute(Context context, MyList operation_chain, long mark_ID) { - for (ExecutionUnit operation : operation_chain) { + public void execute(Context context, MyList operation_chain, long mark_ID) { + for (Operation operation : operation_chain) { execute(operation, mark_ID, false); } } @@ -75,8 +74,8 @@ public void execute(Context context, MyList operation_chain, long * @return */ @Override - protected SchedulingUnit next(Context context) { - SchedulingUnit operationChain = context.ready_oc; + protected OperationChain next(Context context) { + OperationChain operationChain = context.ready_oc; context.ready_oc = null; return operationChain;// if a null is returned, it means, we are done with this level! } @@ -88,7 +87,7 @@ protected SchedulingUnit next(Context context) { * 3. shared: put all operations in a pool and */ @Override - public void DISTRIBUTE(SchedulingUnit task, Context context) { + public void DISTRIBUTE(OperationChain task, Context context) { context.ready_oc = task; } @@ -98,9 +97,9 @@ public void DISTRIBUTE(SchedulingUnit task, Context context) { * @param context * @return */ - protected SchedulingUnit Next(Context context) { - ArrayList ocs = context.OCSCurrentLayer(); // - SchedulingUnit oc = null; + protected OperationChain Next(Context context) { + ArrayList ocs = context.OCSCurrentLayer(); // + OperationChain oc = null; if (ocs != null && context.currentLevelIndex < ocs.size()) { oc = ocs.get(context.currentLevelIndex++); context.scheduledOPs += oc.getOperations().size(); diff --git a/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java b/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java index 02d6a1342..9ef91de5c 100644 --- a/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java @@ -7,7 +7,7 @@ import scheduler.Request; import scheduler.impl.IScheduler; import scheduler.context.op.OPSchedulerContext; -import scheduler.struct.op.AbstractOperation; +import scheduler.struct.AbstractOperation; import scheduler.struct.op.MetaTypes; import scheduler.struct.op.Operation; import scheduler.struct.op.TaskPrecedenceGraph; diff --git a/state-engine/src/main/java/scheduler/impl/op/structured/OPDFSScheduler.java b/state-engine/src/main/java/scheduler/impl/op/structured/OPDFSScheduler.java index d7e095c1a..681f22713 100644 --- a/state-engine/src/main/java/scheduler/impl/op/structured/OPDFSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/op/structured/OPDFSScheduler.java @@ -32,7 +32,7 @@ public void EXPLORE(Context context) { ProcessedToNextLevel(context); op = Next(context); } - while (op != null && op.hasParents()) ; + while (op != null && op.hasParents()); DISTRIBUTE(op, context); } diff --git a/state-engine/src/main/java/scheduler/impl/op/structured/OPSScheduler.java b/state-engine/src/main/java/scheduler/impl/op/structured/OPSScheduler.java index b6e814f28..8a8f46778 100644 --- a/state-engine/src/main/java/scheduler/impl/op/structured/OPSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/op/structured/OPSScheduler.java @@ -107,8 +107,8 @@ protected void NOTIFY(Operation task, Context context) { * @return */ public Operation next(Context context) { - Operation operation = context.ready_oc; - context.ready_oc = null; + Operation operation = context.ready_op; + context.ready_op = null; return operation; } @@ -123,7 +123,7 @@ public Operation next(Context context) { */ @Override protected void DISTRIBUTE(Operation task, Context context) { - context.ready_oc = task; + context.ready_op = task; } /** diff --git a/state-engine/src/main/java/scheduler/signal/oc/OnExecutedSignal.java b/state-engine/src/main/java/scheduler/signal/oc/OnExecutedSignal.java index 28499fe4f..420b64a80 100644 --- a/state-engine/src/main/java/scheduler/signal/oc/OnExecutedSignal.java +++ b/state-engine/src/main/java/scheduler/signal/oc/OnExecutedSignal.java @@ -1,11 +1,11 @@ package scheduler.signal.oc; -import scheduler.struct.og.AbstractOperation; +import scheduler.struct.og.Operation; import scheduler.struct.og.OperationChain; -public class OnExecutedSignal> - extends OperationChainSignal { - public OnExecutedSignal(SchedulingUnit targetOperationChain) { +public class OnExecutedSignal + extends OperationChainSignal { + public OnExecutedSignal(OperationChain targetOperationChain) { super(targetOperationChain); } } diff --git a/state-engine/src/main/java/scheduler/signal/oc/OnHeaderStartAbortHandlingSignal.java b/state-engine/src/main/java/scheduler/signal/oc/OnHeaderStartAbortHandlingSignal.java index ca3c0e221..9dcd66736 100644 --- a/state-engine/src/main/java/scheduler/signal/oc/OnHeaderStartAbortHandlingSignal.java +++ b/state-engine/src/main/java/scheduler/signal/oc/OnHeaderStartAbortHandlingSignal.java @@ -1,18 +1,17 @@ package scheduler.signal.oc; -import scheduler.struct.og.AbstractOperation; +import scheduler.struct.og.Operation; import scheduler.struct.og.OperationChain; -public class OnHeaderStartAbortHandlingSignal> - extends OperationChainSignal { - private final OP abortedOp; +public class OnHeaderStartAbortHandlingSignal extends OperationChainSignal { + private final Operation abortedOp; - public OnHeaderStartAbortHandlingSignal(OC targetOperationChain, OP abortedOp) { + public OnHeaderStartAbortHandlingSignal(OperationChain targetOperationChain, Operation abortedOp) { super(targetOperationChain); this.abortedOp = abortedOp; } - public OP getOperation() { + public Operation getOperation() { return abortedOp; } } diff --git a/state-engine/src/main/java/scheduler/signal/oc/OnNeedAbortHandlingSignal.java b/state-engine/src/main/java/scheduler/signal/oc/OnNeedAbortHandlingSignal.java index e90f89f13..7471476e3 100644 --- a/state-engine/src/main/java/scheduler/signal/oc/OnNeedAbortHandlingSignal.java +++ b/state-engine/src/main/java/scheduler/signal/oc/OnNeedAbortHandlingSignal.java @@ -1,18 +1,17 @@ package scheduler.signal.oc; -import scheduler.struct.og.AbstractOperation; +import scheduler.struct.og.Operation; import scheduler.struct.og.OperationChain; -public class OnNeedAbortHandlingSignal> - extends OperationChainSignal { - private final OP abortedOp; +public class OnNeedAbortHandlingSignal extends OperationChainSignal { + private final Operation abortedOp; - public OnNeedAbortHandlingSignal(OC targetOperationChain, OP abortedOp) { + public OnNeedAbortHandlingSignal(OperationChain targetOperationChain, Operation abortedOp) { super(targetOperationChain); this.abortedOp = abortedOp; } - public OP getOperation() { + public Operation getOperation() { return abortedOp; } } diff --git a/state-engine/src/main/java/scheduler/signal/oc/OnParentExecutedSignal.java b/state-engine/src/main/java/scheduler/signal/oc/OnParentExecutedSignal.java index 5086db764..055ef2c06 100644 --- a/state-engine/src/main/java/scheduler/signal/oc/OnParentExecutedSignal.java +++ b/state-engine/src/main/java/scheduler/signal/oc/OnParentExecutedSignal.java @@ -1,14 +1,13 @@ package scheduler.signal.oc; -import scheduler.struct.og.AbstractOperation; +import scheduler.struct.og.Operation; import scheduler.struct.og.MetaTypes.DependencyType; import scheduler.struct.og.OperationChain; -public class OnParentExecutedSignal> - extends OperationChainSignal { +public class OnParentExecutedSignal extends OperationChainSignal { private final DependencyType dependencyType; - public OnParentExecutedSignal(OC targetOperationChain, DependencyType dependencyType) { + public OnParentExecutedSignal(OperationChain targetOperationChain, DependencyType dependencyType) { super(targetOperationChain); this.dependencyType = dependencyType; } diff --git a/state-engine/src/main/java/scheduler/signal/oc/OnRollbackAndRedoSignal.java b/state-engine/src/main/java/scheduler/signal/oc/OnRollbackAndRedoSignal.java index efa09ddbd..3d9619833 100644 --- a/state-engine/src/main/java/scheduler/signal/oc/OnRollbackAndRedoSignal.java +++ b/state-engine/src/main/java/scheduler/signal/oc/OnRollbackAndRedoSignal.java @@ -1,11 +1,10 @@ package scheduler.signal.oc; -import scheduler.struct.og.AbstractOperation; +import scheduler.struct.og.Operation; import scheduler.struct.og.OperationChain; -public class OnRollbackAndRedoSignal> - extends OperationChainSignal { - public OnRollbackAndRedoSignal(OC targetOperationChain) { +public class OnRollbackAndRedoSignal extends OperationChainSignal { + public OnRollbackAndRedoSignal(OperationChain targetOperationChain) { super(targetOperationChain); } } diff --git a/state-engine/src/main/java/scheduler/signal/oc/OnRootSignal.java b/state-engine/src/main/java/scheduler/signal/oc/OnRootSignal.java index b4e03cbfb..06ec17841 100644 --- a/state-engine/src/main/java/scheduler/signal/oc/OnRootSignal.java +++ b/state-engine/src/main/java/scheduler/signal/oc/OnRootSignal.java @@ -1,11 +1,10 @@ package scheduler.signal.oc; -import scheduler.struct.og.AbstractOperation; +import scheduler.struct.og.Operation; import scheduler.struct.og.OperationChain; -public class OnRootSignal> - extends OperationChainSignal { - public OnRootSignal(OC targetOperationChain) { +public class OnRootSignal extends OperationChainSignal { + public OnRootSignal(OperationChain targetOperationChain) { super(targetOperationChain); } } diff --git a/state-engine/src/main/java/scheduler/signal/oc/OperationChainSignal.java b/state-engine/src/main/java/scheduler/signal/oc/OperationChainSignal.java index dd38d7965..faf924930 100644 --- a/state-engine/src/main/java/scheduler/signal/oc/OperationChainSignal.java +++ b/state-engine/src/main/java/scheduler/signal/oc/OperationChainSignal.java @@ -1,18 +1,17 @@ package scheduler.signal.oc; import scheduler.signal.NotificationSignal; -import scheduler.struct.og.AbstractOperation; +import scheduler.struct.og.Operation; import scheduler.struct.og.OperationChain; -public abstract class OperationChainSignal> - implements NotificationSignal { - private final OC targetOperationChain; +public abstract class OperationChainSignal implements NotificationSignal { + private final OperationChain targetOperationChain; - public OperationChainSignal(OC targetOperationChain) { + public OperationChainSignal(OperationChain targetOperationChain) { this.targetOperationChain = targetOperationChain; } - public OC getTargetOperationChain() { + public OperationChain getTargetOperationChain() { return targetOperationChain; } } diff --git a/state-engine/src/main/java/scheduler/statemanager/og/OperationChainStateListener.java b/state-engine/src/main/java/scheduler/statemanager/og/OperationChainStateListener.java index 02969f001..42211fafe 100644 --- a/state-engine/src/main/java/scheduler/statemanager/og/OperationChainStateListener.java +++ b/state-engine/src/main/java/scheduler/statemanager/og/OperationChainStateListener.java @@ -1,14 +1,13 @@ package scheduler.statemanager.og; -import scheduler.struct.og.AbstractOperation; import scheduler.struct.og.MetaTypes.DependencyType; import scheduler.struct.og.OperationChain; -public interface OperationChainStateListener> { +public interface OperationChainStateListener { - void onOcRootStart(SchedulingUnit operationChain); + void onOcRootStart(OperationChain operationChain); - void onOcExecuted(SchedulingUnit operationChain); + void onOcExecuted(OperationChain operationChain); - void onOcParentExecuted(SchedulingUnit operationChain, DependencyType dependencyType); + void onOcParentExecuted(OperationChain operationChain, DependencyType dependencyType); } diff --git a/state-engine/src/main/java/scheduler/statemanager/og/PartitionStateManager.java b/state-engine/src/main/java/scheduler/statemanager/og/PartitionStateManager.java index cb604b5e1..3b061c9e9 100644 --- a/state-engine/src/main/java/scheduler/statemanager/og/PartitionStateManager.java +++ b/state-engine/src/main/java/scheduler/statemanager/og/PartitionStateManager.java @@ -7,8 +7,7 @@ import scheduler.signal.oc.OnRootSignal; import scheduler.signal.oc.OperationChainSignal; import scheduler.struct.og.MetaTypes.DependencyType; -import scheduler.struct.og.nonstructured.NSOperation; -import scheduler.struct.og.nonstructured.NSOperationChain; +import scheduler.struct.og.OperationChain; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; @@ -16,8 +15,8 @@ /** * Local to every TPGScheduler context. */ -public class PartitionStateManager implements Runnable, OperationChainStateListener { - public final Queue> ocSignalQueue; +public class PartitionStateManager implements Runnable, OperationChainStateListener { + public final Queue ocSignalQueue; private OGNSScheduler.ExecutableTaskListener executableTaskListener; public PartitionStateManager() { @@ -31,25 +30,25 @@ public void run() { } @Override - public void onOcRootStart(NSOperationChain operationChain) { - ocSignalQueue.add(new OnRootSignal<>(operationChain)); + public void onOcRootStart(OperationChain operationChain) { + ocSignalQueue.add(new OnRootSignal(operationChain)); } @Override - public void onOcExecuted(NSOperationChain operationChain) { - ocSignalQueue.add(new OnExecutedSignal<>(operationChain)); + public void onOcExecuted(OperationChain operationChain) { + ocSignalQueue.add(new OnExecutedSignal(operationChain)); } @Override - public void onOcParentExecuted(NSOperationChain operationChain, DependencyType dependencyType) { - ocSignalQueue.add(new OnParentExecutedSignal<>(operationChain, dependencyType)); + public void onOcParentExecuted(OperationChain operationChain, DependencyType dependencyType) { + ocSignalQueue.add(new OnParentExecutedSignal(operationChain, dependencyType)); } public void handleStateTransitions() { - OperationChainSignal ocSignal = ocSignalQueue.poll(); + OperationChainSignal ocSignal = ocSignalQueue.poll(); while (ocSignal != null) { - NSOperationChain operationChain = ocSignal.getTargetOperationChain(); + OperationChain operationChain = ocSignal.getTargetOperationChain(); if (ocSignal instanceof OnRootSignal) { ocRootStartTransition(operationChain); } else if (ocSignal instanceof OnExecutedSignal) { @@ -61,14 +60,14 @@ public void handleStateTransitions() { } } - private void ocRootStartTransition(NSOperationChain operationChain) { + private void ocRootStartTransition(OperationChain operationChain) { executableTaskListener.onOCExecutable(operationChain); } - private void ocExecutedTransition(NSOperationChain operationChain) { + private void ocExecutedTransition(OperationChain operationChain) { operationChain.isExecuted = true; - for (NSOperationChain child : operationChain.getChildren()) { + for (OperationChain child : operationChain.getChildren()) { if (child.ocParentsCount.get() > 0) { ((OGNSContext) child.context).partitionStateManager.onOcParentExecuted(child, DependencyType.FD); } @@ -77,7 +76,7 @@ private void ocExecutedTransition(NSOperationChain operationChain) { } - private void ocParentExecutedTransition(NSOperationChain operationChain) { + private void ocParentExecutedTransition(OperationChain operationChain) { operationChain.updateDependency(); if (!operationChain.hasParents() && !operationChain.isExecuted) { executableTaskListener.onOCExecutable(operationChain); diff --git a/state-engine/src/main/java/scheduler/statemanager/og/PartitionStateManagerWithAbort.java b/state-engine/src/main/java/scheduler/statemanager/og/PartitionStateManagerWithAbort.java index 5864a3622..179d2625c 100644 --- a/state-engine/src/main/java/scheduler/statemanager/og/PartitionStateManagerWithAbort.java +++ b/state-engine/src/main/java/scheduler/statemanager/og/PartitionStateManagerWithAbort.java @@ -4,8 +4,8 @@ import scheduler.impl.og.nonstructured.OGNSAScheduler; import scheduler.signal.oc.*; import scheduler.struct.og.MetaTypes.DependencyType; -import scheduler.struct.og.nonstructured.NSAOperationChain; -import scheduler.struct.og.nonstructured.NSAOperation; +import scheduler.struct.og.Operation; +import scheduler.struct.og.OperationChain; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; @@ -15,8 +15,8 @@ /** * Local to every TPGScheduler context. */ -public class PartitionStateManagerWithAbort implements Runnable, OperationChainStateListener { - public final Queue> ocSignalQueue; +public class PartitionStateManagerWithAbort implements Runnable, OperationChainStateListener { + public final Queue ocSignalQueue; private OGNSAScheduler.ExecutableTaskListener executableTaskListener; public PartitionStateManagerWithAbort() { @@ -30,36 +30,36 @@ public void run() { } @Override - public void onOcRootStart(NSAOperationChain operationChain) { - ocSignalQueue.add(new OnRootSignal<>(operationChain)); + public void onOcRootStart(OperationChain operationChain) { + ocSignalQueue.add(new OnRootSignal(operationChain)); } @Override - public void onOcExecuted(NSAOperationChain operationChain) { - ocSignalQueue.add(new OnExecutedSignal<>(operationChain)); + public void onOcExecuted(OperationChain operationChain) { + ocSignalQueue.add(new OnExecutedSignal(operationChain)); } @Override - public void onOcParentExecuted(NSAOperationChain operationChain, DependencyType dependencyType) { - ocSignalQueue.add(new OnParentExecutedSignal<>(operationChain, dependencyType)); + public void onOcParentExecuted(OperationChain operationChain, DependencyType dependencyType) { + ocSignalQueue.add(new OnParentExecutedSignal(operationChain, dependencyType)); } - public void onHeaderStartAbortHandling(NSAOperationChain operationChain, NSAOperation abortedOp) { - ocSignalQueue.add(new OnHeaderStartAbortHandlingSignal<>(operationChain, abortedOp)); + public void onHeaderStartAbortHandling(OperationChain operationChain, Operation abortedOp) { + ocSignalQueue.add(new OnHeaderStartAbortHandlingSignal(operationChain, abortedOp)); } - public void onOcNeedAbortHandling(NSAOperationChain operationChain, NSAOperation abortedOp) { - ocSignalQueue.add(new OnNeedAbortHandlingSignal<>(operationChain, abortedOp)); + public void onOcNeedAbortHandling(OperationChain operationChain, Operation abortedOp) { + ocSignalQueue.add(new OnNeedAbortHandlingSignal(operationChain, abortedOp)); } - public void onOcRollbackAndRedo(NSAOperationChain operationChain) { - ocSignalQueue.add(new OnRollbackAndRedoSignal<>(operationChain)); + public void onOcRollbackAndRedo(OperationChain operationChain) { + ocSignalQueue.add(new OnRollbackAndRedoSignal(operationChain)); } public void handleStateTransitions() { - OperationChainSignal ocSignal = ocSignalQueue.poll(); + OperationChainSignal ocSignal = ocSignalQueue.poll(); while (ocSignal != null) { - NSAOperationChain operationChain = ocSignal.getTargetOperationChain(); + OperationChain operationChain = ocSignal.getTargetOperationChain(); if (ocSignal instanceof OnRootSignal) { ocRootStartTransition(operationChain); } else if (ocSignal instanceof OnExecutedSignal) { @@ -67,26 +67,26 @@ public void handleStateTransitions() { } else if (ocSignal instanceof OnParentExecutedSignal) { ocParentExecutedTransition(operationChain); } else if (ocSignal instanceof OnNeedAbortHandlingSignal) { - ocAbortHandlingTransition(operationChain, ((OnNeedAbortHandlingSignal) ocSignal).getOperation()); + ocAbortHandlingTransition(operationChain, ((OnNeedAbortHandlingSignal) ocSignal).getOperation()); } else if (ocSignal instanceof OnRollbackAndRedoSignal) { ocRollbackAndRedoTransition(operationChain); } else if (ocSignal instanceof OnHeaderStartAbortHandlingSignal) { - ocHeaderStartAbortHandlingTransition(operationChain, ((OnHeaderStartAbortHandlingSignal) ocSignal).getOperation()); + ocHeaderStartAbortHandlingTransition(operationChain, ((OnHeaderStartAbortHandlingSignal) ocSignal).getOperation()); } ocSignal = ocSignalQueue.poll(); } } - private void ocRootStartTransition(NSAOperationChain operationChain) { + private void ocRootStartTransition(OperationChain operationChain) { executableTaskListener.onOCExecutable(operationChain); } - private void ocExecutedTransition(NSAOperationChain operationChain) { + private void ocExecutedTransition(OperationChain operationChain) { if (!operationChain.needAbortHandling) { operationChain.isExecuted = true; executableTaskListener.onOCFinalized(operationChain); - for (NSAOperationChain child : operationChain.getChildren()) { + for (OperationChain child : operationChain.getChildren()) { if (child.ocParentsCount.get() > 0) { ((OGNSAContext) child.context).partitionStateManager.onOcParentExecuted(child, DependencyType.FD); } @@ -96,10 +96,10 @@ private void ocExecutedTransition(NSAOperationChain operationChain) { } } - private void scheduleAbortHandling(NSAOperationChain operationChain) { + private void scheduleAbortHandling(OperationChain operationChain) { // mark aborted operations and notify OC contains header operation to abort // notify children to rollback and redo if it is executed - for (NSAOperation failedOp : operationChain.failedOperations) { + for (Operation failedOp : operationChain.failedOperations) { ((OGNSAContext) failedOp.getHeader().context).partitionStateManager.onHeaderStartAbortHandling(failedOp.getHeader().getOC(), failedOp.getHeader()); } operationChain.needAbortHandling = false; @@ -107,23 +107,23 @@ private void scheduleAbortHandling(NSAOperationChain operationChain) { executableTaskListener.onOCExecutable(operationChain); } - private void ocHeaderStartAbortHandlingTransition(NSAOperationChain operationChain, NSAOperation header) { + private void ocHeaderStartAbortHandlingTransition(OperationChain operationChain, Operation header) { if (!header.getOperationState().equals(OperationStateType.ABORTED)) { // header might receive multiple abort handling signal, but transaction abort should only happen once. header.stateTransition(OperationStateType.ABORTED); - for (NSAOperation abortedOp : header.getDescendants()) { + for (Operation abortedOp : header.getDescendants()) { ((OGNSAContext) abortedOp.context).partitionStateManager.onOcNeedAbortHandling(abortedOp.getOC(), abortedOp); } } } - private void ocAbortHandlingTransition(NSAOperationChain operationChain, NSAOperation abortedOp) { + private void ocAbortHandlingTransition(OperationChain operationChain, Operation abortedOp) { abortedOp.stateTransition(OperationStateType.ABORTED); assert operationChain.getOperations().contains(abortedOp); if (!abortedOp.isFailed) { // System.out.println(abortedOp); - for (NSAOperation operation : operationChain.getOperations()) { + for (Operation operation : operationChain.getOperations()) { if (!operation.getOperationState().equals(OperationStateType.ABORTED)) { operation.stateTransition(OperationStateType.BLOCKED); } @@ -150,15 +150,15 @@ private void ocAbortHandlingTransition(NSAOperationChain operationChain, NSAOper // } } - private void ocParentExecutedTransition(NSAOperationChain operationChain) { + private void ocParentExecutedTransition(OperationChain operationChain) { operationChain.updateDependency(); if (!operationChain.hasParents() && !operationChain.isExecuted) { executableTaskListener.onOCExecutable(operationChain); } } - private void ocRollbackAndRedoTransition(NSAOperationChain operationChain) { - for (NSAOperation operation : operationChain.getOperations()) { + private void ocRollbackAndRedoTransition(OperationChain operationChain) { + for (Operation operation : operationChain.getOperations()) { if (!operation.getOperationState().equals(OperationStateType.ABORTED)) operation.stateTransition(OperationStateType.BLOCKED); } @@ -170,9 +170,9 @@ private void ocRollbackAndRedoTransition(NSAOperationChain operationChain) { } } - private void notifyChildrenRollbackAndRedo(NSAOperationChain operationChain) { + private void notifyChildrenRollbackAndRedo(OperationChain operationChain) { // notify children to rollback and redo - for (NSAOperationChain child : operationChain.getChildren()) { + for (OperationChain child : operationChain.getChildren()) { ((OGNSAContext) child.context).partitionStateManager.onOcRollbackAndRedo(child); } } diff --git a/state-engine/src/main/java/scheduler/struct/op/AbstractOperation.java b/state-engine/src/main/java/scheduler/struct/AbstractOperation.java similarity index 98% rename from state-engine/src/main/java/scheduler/struct/op/AbstractOperation.java rename to state-engine/src/main/java/scheduler/struct/AbstractOperation.java index 024eccae1..be8133728 100644 --- a/state-engine/src/main/java/scheduler/struct/op/AbstractOperation.java +++ b/state-engine/src/main/java/scheduler/struct/AbstractOperation.java @@ -1,4 +1,4 @@ -package scheduler.struct.op; +package scheduler.struct; import content.common.CommonMetaTypes; import storage.SchemaRecordRef; diff --git a/state-engine/src/main/java/scheduler/struct/og/AbstractOperation.java b/state-engine/src/main/java/scheduler/struct/og/AbstractOperation.java deleted file mode 100644 index 4da6f6b2a..000000000 --- a/state-engine/src/main/java/scheduler/struct/og/AbstractOperation.java +++ /dev/null @@ -1,111 +0,0 @@ -package scheduler.struct.og; - -import content.common.CommonMetaTypes; -import scheduler.struct.op.MetaTypes.OperationStateType; -import storage.SchemaRecordRef; -import storage.TableRecord; -import transaction.context.TxnContext; -import transaction.function.Condition; -import transaction.function.Function; - -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedDeque; - -/** - * TODO: clean ``state" and ``reference". - */ -public abstract class AbstractOperation implements Comparable { - - //required by READ_WRITE_and Condition. - public final Function function; - public final String table_name; - public final String pKey; - public final TxnContext txn_context; - public final CommonMetaTypes.AccessType accessType; - public final TableRecord d_record; - public final long bid; - public volatile SchemaRecordRef record_ref;//required by read-only: the placeholder of the reading d_record. - //only update corresponding column. - //required by READ_WRITE. - public volatile TableRecord s_record;//only if it is different from d_record. - public volatile TableRecord[] condition_records; -// public String[] condition_sourceTable = null; -// public String[] condition_source = null; - public Condition condition; - public int[] success; - private OperationStateType operationState; - public boolean isFailed = false; // whether the operation is failed, this is used to detect transaction abort - -// public volatile AbstractOperation[] fdParentOps; // parent ops that accessing conditioned records and has smaller -// public volatile List fd_parents; // parent ops that accessing conditioned records and has smaller -// public HashMap condition_source_to_index; - - public final Queue fd_parents; // the functional dependencies ops to be executed in advance - - public AbstractOperation(String pKey, Function function, String table_name, SchemaRecordRef record_ref, TableRecord[] condition_records, Condition condition, int[] success, - TxnContext txn_context, CommonMetaTypes.AccessType accessType, TableRecord s_record, TableRecord d_record, long bid) { - this.pKey = pKey; - this.function = function; - this.table_name = table_name; - this.record_ref = record_ref;//this holds events' record_ref. - this.condition_records = condition_records; - this.condition = condition; - this.success = success; - this.txn_context = txn_context; - this.accessType = accessType; - this.s_record = s_record; - this.d_record = d_record; - this.bid = bid; - - - // finctional dependencies, this should be concurrent because cross thread access - fd_parents = new ConcurrentLinkedDeque<>(); // the finctional dependnecies ops to be executed in advance - - operationState = OperationStateType.BLOCKED; - } - - @Override - public String toString() { - return table_name + " " + d_record.record_.GetPrimaryKey() + " " + bid + " " + operationState; - } - - /** - * TODO: make it better. - * It has an assumption that no duplicate keys for the same BID. --> This helps a lot! - * - * @param operation - * @return - */ - @Override - public int compareTo(AbstractOperation operation) { - if (this.bid == (operation.bid)) { - if (!this.table_name.equals(operation.table_name)) { - if (this.table_name.equals("accounts")) - return 1; - else - return -1; - } - return this.d_record.getID() - operation.d_record.getID(); - } else - return Long.compare(this.bid, operation.bid); - } - -// public void setConditionSources(String[] condition_sourceTable, String[] condition_source) { -// this.condition_sourceTable = condition_sourceTable; -// this.condition_source = condition_source; -// } - - - public void addFDParent(AbstractOperation parent) { -// assert condition_source_to_index.containsKey(pKey); - fd_parents.add(parent); - } - - public void stateTransition(OperationStateType state) { - operationState = state; - } - - public OperationStateType getOperationState() { - return operationState; - } -} diff --git a/state-engine/src/main/java/scheduler/struct/og/Operation.java b/state-engine/src/main/java/scheduler/struct/og/Operation.java new file mode 100644 index 000000000..6108da794 --- /dev/null +++ b/state-engine/src/main/java/scheduler/struct/og/Operation.java @@ -0,0 +1,138 @@ +package scheduler.struct.og; + +import content.common.CommonMetaTypes; +import scheduler.context.og.OGSchedulerContext; +import scheduler.struct.AbstractOperation; +import scheduler.struct.op.MetaTypes.OperationStateType; +import storage.SchemaRecordRef; +import storage.TableRecord; +import transaction.context.TxnContext; +import transaction.function.Condition; +import transaction.function.Function; + +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedDeque; + +/** + * TODO: clean ``state" and ``reference". + */ +public class Operation extends AbstractOperation implements Comparable { + + //required by READ_WRITE_and Condition. + public final String pKey; + public final OGSchedulerContext context; + protected final Queue ld_descendant_operations; + //required by READ_WRITE. +// public String[] condition_sourceTable = null; +// public String[] condition_source = null; + private OperationStateType operationState; + public boolean isFailed = false; // whether the operation is failed, this is used to detect transaction abort + +// public volatile Operation[] fdParentOps; // parent ops that accessing conditioned records and has smaller +// public volatile List fd_parents; // parent ops that accessing conditioned records and has smaller +// public HashMap condition_source_to_index; + + public final Queue fd_parents; // the functional dependencies ops to be executed in advance + + private int txnOpId = 0; + // logical dependencies are to be stored for the purpose of abort handling + private Operation ld_head_operation = null; // the logical dependencies ops to be executed after this op. + private OperationChain oc; // used for dependency resolved notification under greedy smart + + + public Operation(String pKey, Function function, String table_name, SchemaRecordRef record_ref, TableRecord[] condition_records, Condition condition, int[] success, + TxnContext txn_context, CommonMetaTypes.AccessType accessType, TableRecord s_record, TableRecord d_record, long bid, Context context) { + super(function, table_name, record_ref, condition_records, condition, success, txn_context, accessType, s_record, d_record, bid); + + this.pKey = pKey; + + // finctional dependencies, this should be concurrent because cross thread access + fd_parents = new ConcurrentLinkedDeque<>(); // the finctional dependnecies ops to be executed in advance + + operationState = OperationStateType.BLOCKED; + this.context = context; + ld_descendant_operations = new ArrayDeque<>(); + } + + @Override + public String toString() { + return bid + "|" + txnOpId + "|" + String.format("%-15s", this.getOperationState()); + } + + /** + * TODO: make it better. + * It has an assumption that no duplicate keys for the same BID. --> This helps a lot! + * + * @param operation + * @return + */ + @Override + public int compareTo(Operation operation) { + if (this.bid == (operation.bid)) { + if (!this.table_name.equals(operation.table_name)) { + if (this.table_name.equals("accounts")) + return 1; + else + return -1; + } + return this.d_record.getID() - operation.d_record.getID(); + } else + return Long.compare(this.bid, operation.bid); + } + +// public void setConditionSources(String[] condition_sourceTable, String[] condition_source) { +// this.condition_sourceTable = condition_sourceTable; +// this.condition_source = condition_source; +// } + + + public void addFDParent(Operation parent) { +// assert condition_source_to_index.containsKey(pKey); + fd_parents.add(parent); + } + + public void stateTransition(OperationStateType state) { + operationState = state; + } + + public OperationStateType getOperationState() { + return operationState; + } + + public void setTxnOpId(int op_id) { + this.txnOpId = op_id; + } + + public int getTxnOpId() { + return txnOpId; + } + + public OperationChain getOC() { + return oc; + } + + public void setOC(OperationChain operationChain) { + this.oc = operationChain; + } + + /*********************************Dependencies setup****************************************/ + + public void addHeader(Operation header) { + ld_head_operation = header; + } + + public void addDescendant(Operation descendant) { +// oc.addDescendant(this, descendant); + ld_descendant_operations.add(descendant); + } + + public Collection getDescendants() { + return ld_descendant_operations; + } + + public Operation getHeader() { + return ld_head_operation; + } +} diff --git a/state-engine/src/main/java/scheduler/struct/og/OperationChain.java b/state-engine/src/main/java/scheduler/struct/og/OperationChain.java index 72602d3ff..bf258de0b 100644 --- a/state-engine/src/main/java/scheduler/struct/og/OperationChain.java +++ b/state-engine/src/main/java/scheduler/struct/og/OperationChain.java @@ -1,5 +1,7 @@ package scheduler.struct.og; +import scheduler.context.og.AbstractOGNSContext; +import scheduler.context.og.OGSchedulerContext; import transaction.impl.ordered.MyList; import utils.lib.ConcurrentHashMap; @@ -11,21 +13,29 @@ * We still call it OperationChain in TPG but with different representation * The OperationChain only tries to maintain a data structure for the ease of temporal dependencies construction. */ -public class OperationChain implements Comparable> { +public class OperationChain implements Comparable { public final String tableName; public final String primaryKey; public final long bid; - protected final MyList operations; + protected final MyList operations; public final AtomicInteger ocParentsCount; // OperationChain -> ChildOp that depend on the parent OC in cur OC - public final ConcurrentHashMap, ExecutionUnit> ocParents; - public final ConcurrentHashMap, ExecutionUnit> ocChildren; + public final ConcurrentHashMap ocParents; + public final ConcurrentHashMap ocChildren; private final ConcurrentLinkedQueue potentialChldrenInfo = new ConcurrentLinkedQueue<>(); public boolean isExecuted = false; // protected TaskPrecedenceGraph tpg; -// private final HashSet> scanedOCs = new HashSet<>(); + public OGSchedulerContext context = null; + public boolean needAbortHandling = false; // The abort handling in GS should be residing in each operation chain + public Queue failedOperations = new ArrayDeque<>(); + + private boolean isDependencyLevelCalculated = false; // we only do this once before executing all OCs. + private int dependencyLevel = -1; + + +// private final HashSet scanedOCs = new HashSet<>(); public OperationChain(String tableName, String primaryKey, long bid) { this.tableName = tableName; @@ -41,18 +51,19 @@ public String getTableName() { return tableName; } - public void addOperation(ExecutionUnit op) { + public void addOperation(Operation op) { operations.add(op); + op.setOC(this); // set OC for op to enable txn abort. } - public void addPotentialFDChildren(OperationChain potentialChildren, ExecutionUnit op) { + public void addPotentialFDChildren(OperationChain potentialChildren, Operation op) { potentialChldrenInfo.add(new PotentialChildrenInfo(potentialChildren, op)); } - public void addParent(ExecutionUnit targetOp, OperationChain parentOC) { - Iterator iterator = parentOC.getOperations().descendingIterator(); // we want to get op with largest bid which is smaller than targetOp bid + public void addParent(Operation targetOp, OperationChain parentOC) { + Iterator iterator = parentOC.getOperations().descendingIterator(); // we want to get op with largest bid which is smaller than targetOp bid while (iterator.hasNext()) { - ExecutionUnit parentOp = iterator.next(); + Operation parentOp = iterator.next(); if (parentOp.bid < targetOp.bid) { // find the exact operation in parent OC that this target OP depends on. // setup dependencies on op level first. targetOp.addFDParent(parentOp); @@ -62,7 +73,7 @@ public void addParent(ExecutionUnit targetOp, OperationChain pare } } - protected void setupDependency(ExecutionUnit targetOp, OperationChain parentOC, ExecutionUnit parentOp) { + protected void setupDependency(Operation targetOp, OperationChain parentOC, Operation parentOp) { // if (circularDetection(targetOp, parentOC, parentOp)) return; assert parentOC.getOperations().size() > 0; if (this.ocParents.putIfAbsent(parentOC, parentOp) == null) { @@ -75,21 +86,21 @@ protected void setupDependency(ExecutionUnit targetOp, OperationChain parentOC, ExecutionUnit parentOp) { +// private boolean circularDetection(Operation targetOp, OperationChain parentOC, Operation parentOp) { // boolean isCircular; // // loop to find the circular // isCircular = isCircular(parentOC); // if (isCircular) { // if circular detected, try to solve circular // // TODO: create a new OC and put all ops after circular OP to the new OC. -// OperationChain newOC = tpg.getNewOC(targetOp.table_name, targetOp.d_record.record_.GetPrimaryKey(), targetOp.bid); -// List opsToMigrate = new ArrayList<>(); -// for (ExecutionUnit op : operations) { +// OperationChain newOC = tpg.getNewOC(targetOp.table_name, targetOp.d_record.record_.GetPrimaryKey(), targetOp.bid); +// List opsToMigrate = new ArrayList<>(); +// for (Operation op : operations) { // if (op.bid >= targetOp.bid) { // opsToMigrate.add(op); // } // } // opsToMigrate.forEach(operations::remove); -// for (ExecutionUnit op : opsToMigrate) { +// for (Operation op : opsToMigrate) { // newOC.addOperation(op); // } // newOC.potentialChldrenInfo.addAll(this.potentialChldrenInfo); // move the potentialChildrenInfo to future @@ -100,21 +111,21 @@ protected void setupDependency(ExecutionUnit targetOp, OperationChain parentOC) { + public boolean isCircular(OperationChain parentOC) { boolean isCircular = false; if (parentOC.ocParents.containsKey(this)) { isCircular = true; } // else { // scanedOCs.clear(); -// Collection> selectedOCs = parentOC.ocParents.keySet(); +// Collection selectedOCs = parentOC.ocParents.keySet(); // isCircular = scanParentOCs(selectedOCs); // } return isCircular; } - public boolean isCircularAffected(HashSet> scannedOCs, HashSet> circularOCs) { - for (OperationChain parent : ocParents.keySet()) { + public boolean isCircularAffected(HashSet scannedOCs, HashSet circularOCs) { + for (OperationChain parent : ocParents.keySet()) { if (!scannedOCs.contains(parent)) { // if the oc is not traversed before, no circular. scannedOCs.add(parent); if (parent.isCircularAffected(scannedOCs, circularOCs)) { @@ -127,9 +138,9 @@ public boolean isCircularAffected(HashSet> scanned return false; } - public void dfs(OperationChain oc, HashSet> affectedOCs) { + public void dfs(OperationChain oc, HashSet affectedOCs) { affectedOCs.add(oc); - for (OperationChain childOC : oc.ocChildren.keySet()) { + for (OperationChain childOC : oc.ocChildren.keySet()) { if (!affectedOCs.contains(childOC)) { dfs(childOC, affectedOCs); } @@ -137,14 +148,14 @@ public void dfs(OperationChain oc, HashSet oc, ArrayDeque> resolvedOC) { +// private void relaxDependencies(OperationChain oc, ArrayDeque resolvedOC) { // // remove all parents, update children set of its parents -//// for (OperationChain parent : oc.ocParents.keySet()) { +//// for (OperationChain parent : oc.ocParents.keySet()) { //// parent.ocChildren.remove(oc); //// } //// oc.ocParentsCount.set(0); //// oc.ocParents.clear(); -// for (OperationChain child : oc.ocChildren.keySet()) { +// for (OperationChain child : oc.ocChildren.keySet()) { // if (!resolvedOC.contains(child)) { // resolvedOC.add(child); // relaxDependencies(child, resolvedOC); @@ -153,8 +164,8 @@ public void dfs(OperationChain oc, HashSet> selectedOCs) { -// for (OperationChain oc : selectedOCs) { +// public boolean scanParentOCs(Collection selectedOCs) { +// for (OperationChain oc : selectedOCs) { // if (!oc.ocParents.isEmpty() && !scanedOCs.contains(oc)) { // scanedOCs.add(oc); // if (oc.ocParents.containsKey(this)) { @@ -168,11 +179,11 @@ public void dfs(OperationChain oc, HashSet> selectedOCs) { +// public boolean checkConnectivity(Collection selectedOCs) { // if (selectedOCs.isEmpty()) { // return true; // } -// for (OperationChain oc : selectedOCs) { +// for (OperationChain oc : selectedOCs) { // if (oc.ocParents.isEmpty()) { // return true; // } else { @@ -182,7 +193,7 @@ public void dfs(OperationChain oc, HashSet processed = new ArrayList<>(); for (PotentialChildrenInfo pChildInfo : potentialChldrenInfo) { @@ -195,7 +206,7 @@ public void checkPotentialFDChildrenOnNewArrival(ExecutionUnit newOp) { processed.clear(); } - public MyList getOperations() { + public MyList getOperations() { return operations; } @@ -208,7 +219,7 @@ public String toString() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - OperationChain that = (OperationChain) o; + OperationChain that = (OperationChain) o; return tableName.equals(that.tableName) && primaryKey.equals(that.primaryKey) && bid == that.bid; } @@ -234,7 +245,7 @@ public Collection getChildren() { return (Collection) ocChildren.keySet(); } - public > boolean hasParents() { + public boolean hasParents() { return ocParentsCount.get() > 0; } @@ -248,10 +259,10 @@ public void reset() { } public class PotentialChildrenInfo implements Comparable { - public OperationChain potentialChildOC; - public ExecutionUnit childOp; + public OperationChain potentialChildOC; + public Operation childOp; - public PotentialChildrenInfo(OperationChain oc, ExecutionUnit op) { + public PotentialChildrenInfo(OperationChain oc, Operation op) { this.potentialChildOC = oc; this.childOp = op; } @@ -267,5 +278,55 @@ public void clear() { operations.clear(); ocParents.clear(); ocChildren.clear(); + // Non-structured data structure clearance + needAbortHandling = false; + failedOperations.clear(); + // Structured data structure clearance + isDependencyLevelCalculated = false; + dependencyLevel = -1; + } + + // ------------------ Non-Structured Methods ------------------------ + public void setContext(AbstractOGNSContext context) { + if (this.context == null) { + this.context = context; + } + } + + public void updateDependency() { + ocParentsCount.decrementAndGet(); + } + + public boolean hasChildren() { + return !ocChildren.isEmpty(); + } + + public void rollbackDependency() { + ocParentsCount.incrementAndGet(); + } + + // ------------------ Structured Methods ------------------------ + public synchronized boolean hasValidDependencyLevel() { + return isDependencyLevelCalculated; + } + + public int getDependencyLevel() { + return dependencyLevel; + } + + public synchronized void updateDependencyLevel() { + if (isDependencyLevelCalculated) + return; + dependencyLevel = 0; + for (OperationChain parent : getParents()) { + if (!parent.hasValidDependencyLevel()) { + parent.updateDependencyLevel(); + } + + if (parent.getDependencyLevel() >= dependencyLevel) { + dependencyLevel = parent.getDependencyLevel() + 1; + } + } + isDependencyLevelCalculated = true; } } diff --git a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java index 8d4219105..201ba3abb 100644 --- a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java @@ -9,7 +9,6 @@ import scheduler.context.og.OGSContext; import scheduler.context.og.OGSchedulerContext; import scheduler.struct.op.MetaTypes; -import scheduler.struct.og.nonstructured.AbstractNSOperationChain; import transaction.impl.ordered.MyList; import utils.AppConfig; import utils.SOURCE_CONTROL; @@ -35,15 +34,15 @@ * | * -> Key: OperationChain [ Operation... ] */ -public class TaskPrecedenceGraph, SchedulingUnit extends OperationChain, ExecutionUnit extends AbstractOperation> { +public class TaskPrecedenceGraph { // all parameters in this class should be thread safe. private static final Logger LOG = LoggerFactory.getLogger(TaskPrecedenceGraph.class); public final Map threadToContextMap; public final int totalThreads; protected final int delta;//range of each partition. depends on the number of op in the stage. private final int NUM_ITEMS; - private final ConcurrentHashMap> operationChains;//shared data structure. - private final ConcurrentHashMap> threadToOCs; + private final ConcurrentHashMap> operationChains;//shared data structure. + private final ConcurrentHashMap> threadToOCs; CyclicBarrier barrier; private int maxLevel = 0; // just for layered scheduling private final int app; @@ -57,7 +56,7 @@ public void reset(Context context) { operationChains.get("bookEntries").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); } // threadToOCs.get(context.thisThreadId).clear(); - for (SchedulingUnit oc : threadToOCs.get(context.thisThreadId)) { + for (OperationChain oc : threadToOCs.get(context.thisThreadId)) { oc.clear(); } // this.setOCs(context); @@ -96,7 +95,7 @@ public TaskPrecedenceGraph(int totalThreads, int delta, int NUM_ITEMS, int app) */ // TODO: if running multiple batches, this will be a problem. public void setOCs(Context context) { - ArrayDeque ocs = new ArrayDeque<>(); + ArrayDeque ocs = new ArrayDeque<>(); int left_bound = context.thisThreadId * delta; int right_bound; if (context.thisThreadId == totalThreads - 1) {//last executor need to handle left-over @@ -108,12 +107,12 @@ public void setOCs(Context context) { for (int key = left_bound; key < right_bound; key++) { _key = String.valueOf(key); if (app == 0) { - SchedulingUnit gsOC = context.createTask("MicroTable", _key, 0); + OperationChain gsOC = context.createTask("MicroTable", _key, 0); operationChains.get("MicroTable").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, gsOC); ocs.add(gsOC); } else if (app == 1) { - SchedulingUnit accOC = context.createTask("accounts", _key, 0); - SchedulingUnit beOC = context.createTask("bookEntries", _key, 0); + OperationChain accOC = context.createTask("accounts", _key, 0); + OperationChain beOC = context.createTask("bookEntries", _key, 0); operationChains.get("accounts").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, accOC); operationChains.get("bookEntries").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, beOC); ocs.add(accOC); @@ -130,18 +129,18 @@ public void setOCs(Context context) { threadToOCs.put(context.thisThreadId, ocs); } - public TableOCs getTableOCs(String table_name) { + public TableOCs getTableOCs(String table_name) { return operationChains.get(table_name); } public boolean isThreadTOCsReady(){return threadToOCs.size()==totalThreads; } - public ConcurrentHashMap> getOperationChains() { + public ConcurrentHashMap> getOperationChains() { return operationChains; } - public void setupOperationTDFD(ExecutionUnit operation, Request request, Context targetContext) { + public void setupOperationTDFD(Operation operation, Request request, Context targetContext) { // TD - SchedulingUnit oc = addOperationToChain(operation, targetContext.thisThreadId); + OperationChain oc = addOperationToChain(operation, targetContext.thisThreadId); // FD if (request.condition_source != null) checkFD(oc, operation, operation.table_name, operation.d_record.record_.GetPrimaryKey(), request.condition_sourceTable, request.condition_source); @@ -153,8 +152,8 @@ public void firstTimeExploreTPG(Context context) { int threadId = context.thisThreadId; MeasureTools.BEGIN_FIRST_EXPLORE_TIME_MEASURE(threadId); // assert context.totalOsToSchedule == ocs.size(); -// Collection> tableOCsList = getOperationChains().values(); -// for (TableOCs tableOCs : tableOCsList) {//for each table. +// Collection> tableOCsList = getOperationChains().values(); +// for (TableOCs tableOCs : tableOCsList) {//for each table. // threadToOCs.computeIfAbsent(threadId, s -> new ArrayDeque<>()).addAll(tableOCs.threadOCsMap.get(threadId).holder_v1.values()); // } @@ -162,24 +161,24 @@ public void firstTimeExploreTPG(Context context) { MeasureTools.END_FIRST_EXPLORE_TIME_MEASURE(threadId); } - private void submit(Context context, Collection ocs) { - HashSet> scannedOCs = new HashSet<>(); - HashSet> circularOCs = new HashSet<>(); - HashSet> resolvedOC = new HashSet<>(); + private void submit(Context context, Collection ocs) { + HashSet scannedOCs = new HashSet<>(); + HashSet circularOCs = new HashSet<>(); + HashSet resolvedOC = new HashSet<>(); // TODO: simple dfs to solve circular, more efficient algorithm need to be involved. keywords: 如何找出有向图中的所有环路？ -// HashMap dfn = new HashMap<>(); -// HashMap low = new HashMap<>(); -// HashMap inStack = new HashMap<>(); -// Stack stack = new Stack<>(); +// HashMap dfn = new HashMap<>(); +// HashMap low = new HashMap<>(); +// HashMap inStack = new HashMap<>(); +// Stack stack = new Stack<>(); // int ts = 1; -// for (SchedulingUnit oc : ocs) { +// for (OperationChain oc : ocs) { // detectCircular(oc, dfn, low, inStack, stack, ts, circularOCs); // } // detectAffectedOCs(scannedOCs, circularOCs); if (AppConfig.isCyclic) { // if the constructed OCs are not cyclic, skip this. circularDetect(context, ocs, scannedOCs, circularOCs, resolvedOC); } -// for (SchedulingUnit oc : ocs) { +// for (OperationChain oc : ocs) { // context.fd += oc.ocParentsCount.get(); // } // LOG.info("id: " + context.thisThreadId + " fd: " + context.fd); @@ -199,13 +198,13 @@ private void submit(Context context, Collection ocs) { } if (enable_log) LOG.info("MaxLevel:" + (((OGSContext) context).maxLevel)); } else if (context instanceof AbstractOGNSContext) { - for (SchedulingUnit oc : ocs) { + for (OperationChain oc : ocs) { if (oc.getOperations().isEmpty()) { continue; } context.totalOsToSchedule += oc.getOperations().size(); context.operationChains.add(oc); - if (!((AbstractNSOperationChain) oc).context.equals(context)) { + if (!((OperationChain) oc).context.equals(context)) { throw new RuntimeException("context of the OC should always be the same as those who submit the OC"); } if (!oc.hasParents()) { @@ -218,8 +217,8 @@ private void submit(Context context, Collection ocs) { if (enable_log) LOG.info("average length of oc:" + context.totalOsToSchedule / ocs.size()); } - private void circularDetect(Context context, Collection ocs, HashSet> scannedOCs, HashSet> circularOCs, HashSet> resolvedOC) { - for (SchedulingUnit oc : ocs) { + private void circularDetect(Context context, Collection ocs, HashSet scannedOCs, HashSet circularOCs, HashSet resolvedOC) { + for (OperationChain oc : ocs) { if (!oc.getOperations().isEmpty()) { context.fd += oc.ocParentsCount.get(); detectAffectedOCs(scannedOCs, circularOCs, oc); @@ -227,7 +226,7 @@ private void circularDetect(Context context, Collection ocs, Has } SOURCE_CONTROL.getInstance().waitForOtherThreads(); // wait until all threads find the circular ocs. int counter = 0; - for (OperationChain oc : circularOCs) { + for (OperationChain oc : circularOCs) { if (Integer.parseInt(oc.primaryKey) / delta == context.thisThreadId) { oc.ocParentsCount.set(0); oc.ocParents.clear(); @@ -242,7 +241,7 @@ private void circularDetect(Context context, Collection ocs, Has public void secondTimeExploreTPG(Context context) { context.redo(); - for (SchedulingUnit oc : threadToOCs.get(context.thisThreadId)) { + for (OperationChain oc : threadToOCs.get(context.thisThreadId)) { if (!oc.getOperations().isEmpty()) { resetOC(oc); } @@ -252,9 +251,9 @@ public void secondTimeExploreTPG(Context context) { if (context instanceof OGSContext) { if (enable_log) LOG.info("MaxLevel:" + (((OGSContext) context).maxLevel)); } else if (context instanceof OGNSContext) { - for (SchedulingUnit oc : threadToOCs.get(context.thisThreadId)) { + for (OperationChain oc : threadToOCs.get(context.thisThreadId)) { if (!oc.getOperations().isEmpty()) { - if (!((AbstractNSOperationChain) oc).context.equals(context)) { + if (!((OperationChain) oc).context.equals(context)) { throw new RuntimeException("context of the OC should always be the same as those who submit the OC"); } if (!oc.hasParents()) { @@ -268,9 +267,9 @@ public void secondTimeExploreTPG(Context context) { MeasureTools.END_FIRST_EXPLORE_TIME_MEASURE(context.thisThreadId); } - private void resetOC(SchedulingUnit oc) { + private void resetOC(OperationChain oc) { oc.reset(); - for (ExecutionUnit op : oc.getOperations()) { + for (Operation op : oc.getOperations()) { op.stateTransition(MetaTypes.OperationStateType.BLOCKED); if (op.isFailed) { // transit state to aborted. op.stateTransition(MetaTypes.OperationStateType.ABORTED); @@ -282,8 +281,8 @@ public void Explore(Context context) { int threadId = context.thisThreadId; MeasureTools.BEGIN_FIRST_EXPLORE_TIME_MEASURE(threadId); // assert context.totalOsToSchedule == ocs.size(); -// Collection> tableOCsList = getOperationChains().values(); -// for (TableOCs tableOCs : tableOCsList) {//for each table. +// Collection> tableOCsList = getOperationChains().values(); +// for (TableOCs tableOCs : tableOCsList) {//for each table. // threadToOCs.computeIfAbsent(threadId, s -> new ArrayDeque<>()).addAll(tableOCs.threadOCsMap.get(threadId).holder_v1.values()); // } @@ -291,10 +290,10 @@ public void Explore(Context context) { MeasureTools.END_FIRST_EXPLORE_TIME_MEASURE(context.thisThreadId); } - private void Submit(Context context, Collection ocs) { - ArrayDeque nonNullOCs = new ArrayDeque<>(); - HashSet> circularOCs = new HashSet<>(); - for (SchedulingUnit oc : ocs) { + private void Submit(Context context, Collection ocs) { + ArrayDeque nonNullOCs = new ArrayDeque<>(); + HashSet circularOCs = new HashSet<>(); + for (OperationChain oc : ocs) { if (!oc.getOperations().isEmpty()) { nonNullOCs.add(oc); // context.fd += oc.ocParentsCount.get(); @@ -303,7 +302,7 @@ private void Submit(Context context, Collection ocs) { } SOURCE_CONTROL.getInstance().waitForOtherThreads(); // wait until all threads find the circular ocs. int counter = 0; - for (OperationChain oc : circularOCs) { + for (OperationChain oc : circularOCs) { if (Integer.parseInt(oc.primaryKey) / delta == context.thisThreadId) { counter ++; oc.ocParentsCount.set(0); @@ -314,10 +313,10 @@ private void Submit(Context context, Collection ocs) { if (enable_log) LOG.info(context.thisThreadId + " : " + counter); // LOG.info("fd number: " + context.fd); assert context instanceof AbstractOGNSContext; - for (SchedulingUnit oc : nonNullOCs) { + for (OperationChain oc : nonNullOCs) { context.totalOsToSchedule += oc.getOperations().size(); context.operationChains.add(oc); - if (!((AbstractNSOperationChain) oc).context.equals(context)) { + if (!((OperationChain) oc).context.equals(context)) { throw new RuntimeException("context of the OC should always be the same as those who submit the OC"); } if (!oc.hasParents()) { @@ -335,11 +334,11 @@ public void ReExplore(Context context) { MeasureTools.END_FIRST_EXPLORE_TIME_MEASURE(context.thisThreadId); } - private void ReSubmit(Context context, Collection ocs) { + private void ReSubmit(Context context, Collection ocs) { context.redo(); - ArrayDeque nonNullOCs = new ArrayDeque<>(); - HashSet> circularOCs = new HashSet<>(); - for (SchedulingUnit oc : ocs) { + ArrayDeque nonNullOCs = new ArrayDeque<>(); + HashSet circularOCs = new HashSet<>(); + for (OperationChain oc : ocs) { if (!oc.getOperations().isEmpty()) { resetOC(oc); nonNullOCs.add(oc); @@ -348,9 +347,9 @@ private void ReSubmit(Context context, Collection ocs) { } SOURCE_CONTROL.getInstance().waitForOtherThreads(); // wait until all threads find the circular ocs. assert context instanceof AbstractOGNSContext; - for (SchedulingUnit oc : nonNullOCs) { + for (OperationChain oc : nonNullOCs) { context.operationChains.add(oc); - if (!((AbstractNSOperationChain) oc).context.equals(context)) { + if (!((OperationChain) oc).context.equals(context)) { throw new RuntimeException("context of the OC should always be the same as those who submit the OC"); } if (!oc.hasParents()) { @@ -360,34 +359,34 @@ private void ReSubmit(Context context, Collection ocs) { if (enable_log) LOG.info("average length of oc:" + context.totalOsToSchedule / ocs.size()); } - private void detectCircular(SchedulingUnit oc, - HashMap dfn, - HashMap low, - HashMap inStack, - Stack stack, + private void detectCircular(OperationChain oc, + HashMap dfn, + HashMap low, + HashMap inStack, + Stack stack, int ts, - HashSet> circularOCs) { + HashSet circularOCs) { if (!oc.getOperations().isEmpty() && !dfn.containsKey(oc)) { tarjanDfs(oc, dfn, low, inStack, stack, ts, circularOCs); } } - private void tarjanDfs(SchedulingUnit oc, - HashMap dfn, - HashMap low, - HashMap inStack, - Stack stack, + private void tarjanDfs(OperationChain oc, + HashMap dfn, + HashMap low, + HashMap inStack, + Stack stack, int ts, - HashSet> circularOCs) { + HashSet circularOCs) { dfn.put(oc, ts); low.put(oc, ts); ts++; stack.push(oc); inStack.put(oc, true); - for (OperationChain parentOC : oc.ocParents.keySet()) { + for (OperationChain parentOC : oc.ocParents.keySet()) { if (!dfn.containsKey(parentOC)) { - tarjanDfs((SchedulingUnit) parentOC, dfn, low, inStack, stack, ts, circularOCs); + tarjanDfs((OperationChain) parentOC, dfn, low, inStack, stack, ts, circularOCs); low.put(oc, Math.min(low.get(oc), low.get(parentOC))); } else if (inStack.get(parentOC)) { low.put(oc, Math.min(low.get(oc), dfn.get(parentOC))); @@ -395,8 +394,8 @@ private void tarjanDfs(SchedulingUnit oc, } if (dfn.get(oc).equals(low.get(oc))) { - SchedulingUnit tmp; - List scc = new ArrayList<>(); + OperationChain tmp; + List scc = new ArrayList<>(); do { tmp = stack.pop(); inStack.put(tmp, false); @@ -413,24 +412,24 @@ private void tarjanDfs(SchedulingUnit oc, * @param affectedOCs * @param circularOCs */ - private void detectAffectedOCs(HashSet> affectedOCs, HashSet> circularOCs) { - for (OperationChain circularOC : circularOCs) { - dfs((SchedulingUnit) circularOC, affectedOCs); + private void detectAffectedOCs(HashSet affectedOCs, HashSet circularOCs) { + for (OperationChain circularOC : circularOCs) { + dfs((OperationChain) circularOC, affectedOCs); } } - public void dfs(SchedulingUnit oc, HashSet> affectedOCs) { + public void dfs(OperationChain oc, HashSet affectedOCs) { affectedOCs.add(oc); - for (OperationChain childOC : oc.ocChildren.keySet()) { + for (OperationChain childOC : oc.ocChildren.keySet()) { if (!affectedOCs.contains(childOC)) { - dfs((SchedulingUnit) childOC, affectedOCs); + dfs((OperationChain) childOC, affectedOCs); } } } - private void detectAffectedOCs(HashSet> scannedOCs, - HashSet> circularOCs, - SchedulingUnit oc) { + private void detectAffectedOCs(HashSet scannedOCs, + HashSet circularOCs, + OperationChain oc) { if (oc.hasParents()) { if (circularOCs.contains(oc)) { return; @@ -445,37 +444,37 @@ private void detectAffectedOCs(HashSet> scannedOCs } } - public SchedulingUnit addOperationToChain(ExecutionUnit operation, int targetThreadId) { + public OperationChain addOperationToChain(Operation operation, int targetThreadId) { // DD: Get the Holder for the table, then get a map for each thread, then get the list of operations String table_name = operation.table_name; String primaryKey = operation.d_record.record_.GetPrimaryKey(); - SchedulingUnit retOc = getOC(table_name, primaryKey, targetThreadId); + OperationChain retOc = getOC(table_name, primaryKey, targetThreadId); retOc.addOperation(operation); return retOc; } - private SchedulingUnit getOC(String tableName, String pKey, int threadId) { - ConcurrentHashMap holder = getTableOCs(tableName).threadOCsMap.get(threadId).holder_v1; + private OperationChain getOC(String tableName, String pKey, int threadId) { + ConcurrentHashMap holder = getTableOCs(tableName).threadOCsMap.get(threadId).holder_v1; return holder.computeIfAbsent(pKey, s -> threadToContextMap.get(threadId).createTask(tableName, pKey, 0)); // return holder.get(pKey); } - private SchedulingUnit getOC(String tableName, String pKey) { + private OperationChain getOC(String tableName, String pKey) { int threadId = Integer.parseInt(pKey) / delta; - ConcurrentHashMap holder = getTableOCs(tableName).threadOCsMap.get(threadId).holder_v1; + ConcurrentHashMap holder = getTableOCs(tableName).threadOCsMap.get(threadId).holder_v1; return holder.computeIfAbsent(pKey, s -> threadToContextMap.get(threadId).createTask(tableName, pKey, 0)); // return holder.get(pKey); } - private void checkFD(SchedulingUnit curOC, ExecutionUnit op, String table_name, + private void checkFD(OperationChain curOC, Operation op, String table_name, String key, String[] condition_sourceTable, String[] condition_source) { if (condition_source != null) { for (int index = 0; index < condition_source.length; index++) { if (table_name.equals(condition_sourceTable[index]) && key.equals(condition_source[index])) continue;// no need to check data dependency on a key itself. - SchedulingUnit OCFromConditionSource = getOC(condition_sourceTable[index], condition_source[index]); + OperationChain OCFromConditionSource = getOC(condition_sourceTable[index], condition_source[index]); // dependency.getOperations().first().bid >= bid -- Check if checking only first ops bid is enough. - MyList conditionedOps = OCFromConditionSource.getOperations(); + MyList conditionedOps = OCFromConditionSource.getOperations(); if (OCFromConditionSource.getOperations().isEmpty() || conditionedOps.first().bid >= op.bid) { OCFromConditionSource.addPotentialFDChildren(curOC, op); } else { diff --git a/state-engine/src/main/java/scheduler/struct/og/nonstructured/AbstractNSOperationChain.java b/state-engine/src/main/java/scheduler/struct/og/nonstructured/AbstractNSOperationChain.java deleted file mode 100644 index 6ac4158f6..000000000 --- a/state-engine/src/main/java/scheduler/struct/og/nonstructured/AbstractNSOperationChain.java +++ /dev/null @@ -1,34 +0,0 @@ -package scheduler.struct.og.nonstructured; - -import scheduler.context.og.AbstractOGNSContext; -import scheduler.struct.og.OperationChain; - -/** - * We still call it OperationChain in TPG but with different representation - * The OperationChain only tries to maintain a data structure for the ease of temporal dependencies construction. - */ -public abstract class AbstractNSOperationChain extends OperationChain { - public AbstractOGNSContext context = null; - - public AbstractNSOperationChain(String tableName, String primaryKey, long bid) { - super(tableName, primaryKey, bid); - } - - public void setContext(AbstractOGNSContext context) { - if (this.context == null) { - this.context = context; - } - } - - public void updateDependency() { - ocParentsCount.decrementAndGet(); - } - - public boolean hasChildren() { - return !ocChildren.isEmpty(); - } - - public void rollbackDependency() { - ocParentsCount.incrementAndGet(); - } -} diff --git a/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSAOperation.java b/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSAOperation.java deleted file mode 100644 index 9f561c61e..000000000 --- a/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSAOperation.java +++ /dev/null @@ -1,108 +0,0 @@ -package scheduler.struct.og.nonstructured; - -import content.common.CommonMetaTypes; -import scheduler.context.og.AbstractOGNSContext; -import storage.SchemaRecordRef; -import storage.TableRecord; -import transaction.context.TxnContext; -import transaction.function.Condition; -import transaction.function.Function; - -import java.util.ArrayDeque; -import java.util.Collection; -import java.util.Queue; - -/** - * contains the place-holder to fill, as well as timestamp (counter). - */ -public class NSAOperation extends NSOperation { - public final AbstractOGNSContext context; - public int txnOpId = 0; - // logical dependencies are to be stored for the purpose of abort handling - private NSAOperation ld_head_operation = null; // the logical dependencies ops to be executed after this op. - private final Queue ld_descendant_operations; - private NSAOperationChain oc; // used for dependency resolved notification under greedy smart - - public NSAOperation(String pKey, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, SchemaRecordRef record_ref) { - this(pKey, null, table_name, txn_context, bid, accessType, record, record_ref, null, null, null, null); - } - - /****************************Defined by MYC*************************************/ - - public NSAOperation(String pKey, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, - Function function, Condition condition, int[] success) { - this(pKey, null, table_name, txn_context, bid, accessType, record, null, function, condition, null, success); - } - - public NSAOperation(String pKey, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, - SchemaRecordRef record_ref, Function function, Condition condition, int[] success) { - this(pKey, null, table_name, txn_context, bid, accessType, record, record_ref, function, condition, null, success); - } - - - public NSAOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, Function function, Condition condition, TableRecord[] condition_records, int[] success) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, null, function, condition, condition_records, success); - } - - public NSAOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, null, null, null, null, null); - } - - public NSAOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, - SchemaRecordRef record_ref) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, null, null, null, null); - } - public NSAOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, - SchemaRecordRef record_ref,Function function) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, function, null, null, null); - } - - public NSAOperation( - String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord record, - SchemaRecordRef record_ref, Function function, Condition condition, - TableRecord[] condition_records, int[] success) { - super(pKey, context, table_name, txn_context, bid, accessType, record, record_ref, function, condition, condition_records, success); - this.context = context; - ld_descendant_operations = new ArrayDeque<>(); - } - - public NSAOperationChain getOC() { - return oc; - } - - public void setOC(NSAOperationChain operationChain) { - this.oc = operationChain; - } - - public int getTxnOpId() { - return txnOpId; - } - - public void setTxnOpId(int txnOpId) { - this.txnOpId = txnOpId; - } - - /*********************************Dependencies setup****************************************/ - - public void addHeader(NSAOperation header) { - ld_head_operation = header; - } - - public void addDescendant(NSAOperation descendant) { -// oc.addDescendant(this, descendant); - ld_descendant_operations.add(descendant); - } - - public Collection getDescendants() { - return ld_descendant_operations; - } - - public NSAOperation getHeader() { - return ld_head_operation; - } -} \ No newline at end of file diff --git a/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSAOperationChain.java b/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSAOperationChain.java deleted file mode 100644 index d285c11ed..000000000 --- a/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSAOperationChain.java +++ /dev/null @@ -1,37 +0,0 @@ -package scheduler.struct.og.nonstructured; - -import java.util.ArrayDeque; -import java.util.Collection; -import java.util.Queue; - -/** - * We still call it OperationChain in TPG but with different representation - * The OperationChain only tries to maintain a data structure for the ease of temporal dependencies construction. - */ -public class NSAOperationChain extends AbstractNSOperationChain { - // maintains descendants of header operation in the oc. HeaderOperation -> Queue - public boolean needAbortHandling = false; // The abort handling in GS should be residing in each operation chain - public Queue failedOperations = new ArrayDeque<>(); - - - public NSAOperationChain(String tableName, String primaryKey, long bid) { - super(tableName, primaryKey, bid); - } - - public void addOperation(NSAOperation op) { - operations.add(op); - op.setOC(this); // set OC for op to enable txn abort. - } - - @Override - public Collection getChildren() { - return super.getChildren(); - } - - @Override - public void clear() { - super.clear(); - needAbortHandling = false; - failedOperations.clear(); - } -} diff --git a/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSOperation.java b/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSOperation.java deleted file mode 100644 index fd8662e5c..000000000 --- a/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSOperation.java +++ /dev/null @@ -1,63 +0,0 @@ -package scheduler.struct.og.nonstructured; - -import content.common.CommonMetaTypes; -import scheduler.context.og.AbstractOGNSContext; -import scheduler.struct.og.AbstractOperation; -import storage.SchemaRecordRef; -import storage.TableRecord; -import transaction.context.TxnContext; -import transaction.function.Condition; -import transaction.function.Function; - -/** - * contains the place-holder to fill, as well as timestamp (counter). - */ -public class NSOperation extends AbstractOperation { - - public NSOperation(String pKey, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, SchemaRecordRef record_ref) { - this(pKey, null, table_name, txn_context, bid, accessType, record, record_ref, null, null, null, null); - } - - /****************************Defined by MYC*************************************/ - - public NSOperation(String pKey, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, - Function function, Condition condition, int[] success) { - this(pKey, null, table_name, txn_context, bid, accessType, record, null, function, condition, null, success); - } - - public NSOperation(String pKey, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, - SchemaRecordRef record_ref, Function function, Condition condition, int[] success) { - this(pKey, null, table_name, txn_context, bid, accessType, record, record_ref, function, condition, null, success); - } - - - public NSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, Function function, Condition condition, TableRecord[] condition_records, int[] success) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, null, function, condition, condition_records, success); - } - - public NSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, null, null, null, null, null); - } - - public NSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, - SchemaRecordRef record_ref) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, null, null, null, null); - } - public NSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, - SchemaRecordRef record_ref,Function function) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, function, null, null, null); - } - - - public NSOperation( - String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord record, - SchemaRecordRef record_ref, Function function, Condition condition, - TableRecord[] condition_records, int[] success) { - super(pKey, function, table_name, record_ref, condition_records, condition, success, txn_context, accessType, record, record, bid); - } -} \ No newline at end of file diff --git a/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSOperationChain.java b/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSOperationChain.java deleted file mode 100644 index bb811d3b9..000000000 --- a/state-engine/src/main/java/scheduler/struct/og/nonstructured/NSOperationChain.java +++ /dev/null @@ -1,18 +0,0 @@ -package scheduler.struct.og.nonstructured; - -import java.util.Collection; - -/** - * We still call it OperationChain in TPG but with different representation - * The OperationChain only tries to maintain a data structure for the ease of temporal dependencies construction. - */ -public class NSOperationChain extends AbstractNSOperationChain { - public NSOperationChain(String tableName, String primaryKey, long bid) { - super(tableName, primaryKey, bid); - } - - @Override - public Collection getChildren() { - return super.getChildren(); - } -} diff --git a/state-engine/src/main/java/scheduler/struct/og/structured/SOperationChain.java b/state-engine/src/main/java/scheduler/struct/og/structured/SOperationChain.java deleted file mode 100644 index c2f940f58..000000000 --- a/state-engine/src/main/java/scheduler/struct/og/structured/SOperationChain.java +++ /dev/null @@ -1,56 +0,0 @@ -package scheduler.struct.og.structured; - -import scheduler.struct.og.AbstractOperation; -import scheduler.struct.og.OperationChain; - -import java.util.Collection; - -/** - * We still call it OperationChain in TPG but with different representation - * The OperationChain only tries to maintain a data structure for the ease of temporal dependencies construction. - */ -public abstract class SOperationChain extends OperationChain { - - private boolean isDependencyLevelCalculated = false; // we only do this once before executing all OCs. - private int dependencyLevel = -1; - - public SOperationChain(String tableName, String primaryKey, long bid) { - super(tableName, primaryKey, bid); - } - - @Override - public Collection getParents() { - return super.getParents(); - } - - public synchronized boolean hasValidDependencyLevel() { - return isDependencyLevelCalculated; - } - - public int getDependencyLevel() { - return dependencyLevel; - } - - public synchronized void updateDependencyLevel() { - if (isDependencyLevelCalculated) - return; - dependencyLevel = 0; - for (SOperationChain parent : getParents()) { - if (!parent.hasValidDependencyLevel()) { - parent.updateDependencyLevel(); - } - - if (parent.getDependencyLevel() >= dependencyLevel) { - dependencyLevel = parent.getDependencyLevel() + 1; - } - } - isDependencyLevelCalculated = true; - } - - @Override - public void clear() { - super.clear(); - isDependencyLevelCalculated = false; - dependencyLevel = -1; - } -} diff --git a/state-engine/src/main/java/scheduler/struct/og/structured/bfs/BFSOperation.java b/state-engine/src/main/java/scheduler/struct/og/structured/bfs/BFSOperation.java deleted file mode 100644 index 5ead2d638..000000000 --- a/state-engine/src/main/java/scheduler/struct/og/structured/bfs/BFSOperation.java +++ /dev/null @@ -1,62 +0,0 @@ -package scheduler.struct.og.structured.bfs; - -import content.common.CommonMetaTypes; -import scheduler.context.og.OGSchedulerContext; -import scheduler.struct.og.AbstractOperation; -import storage.SchemaRecordRef; -import storage.TableRecord; -import transaction.context.TxnContext; -import transaction.function.Condition; -import transaction.function.Function; - -/** - * contains the place-holder to fill, as well as timestamp (counter). - */ -public class BFSOperation extends AbstractOperation { - - public BFSOperation(String pKey, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, SchemaRecordRef record_ref) { - this(pKey, null, table_name, txn_context, bid, accessType, record, record_ref, null, null, null, null); - } - - /****************************Defined by MYC*************************************/ - - public BFSOperation(String pKey, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, - Function function, Condition condition, int[] success) { - this(pKey, null, table_name, txn_context, bid, accessType, record, null, function, condition, null, success); - } - - public BFSOperation(String pKey, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, - SchemaRecordRef record_ref, Function function, Condition condition, int[] success) { - this(pKey, null, table_name, txn_context, bid, accessType, record, record_ref, function, condition, null, success); - } - - - public BFSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, Function function, Condition condition, TableRecord[] condition_records, int[] success) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, null, function, condition, condition_records, success); - } - - public BFSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, null, null, null, null, null); - } - - public BFSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, - SchemaRecordRef record_ref) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, null, null, null, null); - } - public BFSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, - SchemaRecordRef record_ref,Function function) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, function, null, null, null); - } - - public BFSOperation( - String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord record, - SchemaRecordRef record_ref, Function function, Condition condition, - TableRecord[] condition_records, int[] success) { - super(pKey, function, table_name, record_ref, condition_records, condition, success, txn_context, accessType, record, record, bid); - } -} \ No newline at end of file diff --git a/state-engine/src/main/java/scheduler/struct/og/structured/bfs/BFSOperationChain.java b/state-engine/src/main/java/scheduler/struct/og/structured/bfs/BFSOperationChain.java deleted file mode 100644 index ee35fbdaf..000000000 --- a/state-engine/src/main/java/scheduler/struct/og/structured/bfs/BFSOperationChain.java +++ /dev/null @@ -1,14 +0,0 @@ -package scheduler.struct.og.structured.bfs; - -import scheduler.struct.og.structured.SOperationChain; - -/** - * We still call it OperationChain in TPG but with different representation - * The OperationChain only tries to maintain a data structure for the ease of temporal dependencies construction. - */ -public class BFSOperationChain extends SOperationChain { - - public BFSOperationChain(String tableName, String primaryKey, long bid) { - super(tableName, primaryKey, bid); - } -} diff --git a/state-engine/src/main/java/scheduler/struct/og/structured/dfs/DFSOperation.java b/state-engine/src/main/java/scheduler/struct/og/structured/dfs/DFSOperation.java deleted file mode 100644 index 230f7db7d..000000000 --- a/state-engine/src/main/java/scheduler/struct/og/structured/dfs/DFSOperation.java +++ /dev/null @@ -1,79 +0,0 @@ -package scheduler.struct.og.structured.dfs; - -import content.common.CommonMetaTypes; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import scheduler.context.og.OGSchedulerContext; -import scheduler.struct.og.AbstractOperation; -import storage.SchemaRecordRef; -import storage.TableRecord; -import transaction.context.TxnContext; -import transaction.function.Condition; -import transaction.function.Function; - -/** - * contains the place-holder to fill, as well as timestamp (counter). - */ -public class DFSOperation extends AbstractOperation { - private static final Logger LOG = LoggerFactory.getLogger(AbstractOperation.class); - - // operation id under a transaction. - // an operation id to indicate how many operations in front of this operation in the same transaction. - public int txn_op_id = 0; - public String name; - - private DFSOperationChain oc; // used for dependency resolved notification under greedy smart - - public DFSOperation(String pKey, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, SchemaRecordRef record_ref) { - this(pKey, null, table_name, txn_context, bid, accessType, record, record_ref, null, null, null, null); - } - - /****************************Defined by MYC*************************************/ - - public DFSOperation(String pKey, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, - Function function, Condition condition, int[] success) { - this(pKey, null, table_name, txn_context, bid, accessType, record, null, function, condition, null, success); - } - - public DFSOperation(String pKey, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, - SchemaRecordRef record_ref, Function function, Condition condition, int[] success) { - this(pKey, null, table_name, txn_context, bid, accessType, record, record_ref, function, condition, null, success); - } - - - public DFSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, Function function, Condition condition, TableRecord[] condition_records, int[] success) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, null, function, condition, condition_records, success); - } - - public DFSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, null, null, null, null, null); - } - - public DFSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, - SchemaRecordRef record_ref) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, null, null, null, null); - } - public DFSOperation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, - SchemaRecordRef record_ref,Function function) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, function, null, null, null); - } - - public DFSOperation( - String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord record, - SchemaRecordRef record_ref, Function function, Condition condition, - TableRecord[] condition_records, int[] success) { - super(pKey, function, table_name, record_ref, condition_records, condition, success, txn_context, accessType, record, record, bid); - } - - - /****************************End*************************************/ - - public int getTxn_op_id() { - return txn_op_id; - } -} \ No newline at end of file diff --git a/state-engine/src/main/java/scheduler/struct/og/structured/dfs/DFSOperationChain.java b/state-engine/src/main/java/scheduler/struct/og/structured/dfs/DFSOperationChain.java deleted file mode 100644 index 2c2fef494..000000000 --- a/state-engine/src/main/java/scheduler/struct/og/structured/dfs/DFSOperationChain.java +++ /dev/null @@ -1,29 +0,0 @@ -package scheduler.struct.og.structured.dfs; - -import scheduler.struct.og.structured.SOperationChain; - -import java.util.Collection; - -/** - * We still call it OperationChain in TPG but with different representation - * The operationchain only tries to maintain a data structure for the ease of temporal dependencies construction. - */ -public class DFSOperationChain extends SOperationChain { - - public DFSOperationChain(String tableName, String primaryKey, long bid) { - super(tableName, primaryKey, bid); - } - - @Override - public Collection getChildren() { - return super.getChildren(); - } - - public void updateDependency() { - ocParentsCount.decrementAndGet(); - } - - public void rollbackDependency() { - ocParentsCount.incrementAndGet(); - } -} diff --git a/state-engine/src/main/java/scheduler/struct/op/Operation.java b/state-engine/src/main/java/scheduler/struct/op/Operation.java index d0ab979eb..4571ee1f0 100644 --- a/state-engine/src/main/java/scheduler/struct/op/Operation.java +++ b/state-engine/src/main/java/scheduler/struct/op/Operation.java @@ -6,6 +6,7 @@ import scheduler.context.op.OPSContext; import scheduler.context.op.OPSchedulerContext; import scheduler.signal.op.OnParentUpdatedSignal; +import scheduler.struct.AbstractOperation; import scheduler.struct.op.MetaTypes.DependencyType; import scheduler.struct.op.MetaTypes.OperationStateType; import storage.SchemaRecordRef; @@ -18,6 +19,7 @@ import java.util.Deque; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.atomic.AtomicInteger; /** * contains the place-holder to fill, as well as timestamp (counter). @@ -42,7 +44,7 @@ public class Operation extends AbstractOperation implements Comparable Operation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord d_record, Function function, Condition condition, TableRecord[] condition_records, int[] success) { this(pKey, context, table_name, txn_context, bid, accessType, d_record, null, function, condition, condition_records, success); } - public Operation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, null, null, null, null, null); - } - - public Operation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, - SchemaRecordRef record_ref) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, null, null, null, null); - } - public Operation(String pKey, Context context, String table_name, TxnContext txn_context, long bid, - CommonMetaTypes.AccessType accessType, TableRecord d_record, - SchemaRecordRef record_ref,Function function) { - this(pKey, context, table_name, txn_context, bid, accessType, d_record, record_ref, function, null, null, null); - } - public Operation( String pKey, Context context, String table_name, TxnContext txn_context, long bid, CommonMetaTypes.AccessType accessType, TableRecord record, @@ -149,19 +120,19 @@ public int compareTo(Operation operation) { @Override public String toString() { - return bid + "|" + txn_op_id + "|" + String.format("%-15s", this.getOperationState()); + return bid + "|" + txnOpId + "|" + String.format("%-15s", this.getOperationState()); } public void setTxnOpId(int op_id) { - this.txn_op_id = op_id; + this.txnOpId = op_id; } public int getTxnOpId() { - return txn_op_id; + return txnOpId; } public boolean isRoot() { - return operationMetadata.td_countdown == 0 && operationMetadata.fd_countdown == 0 && operationMetadata.ld_countdown == 0; + return operationMetadata.td_countdown.get() == 0 && operationMetadata.fd_countdown.get() == 0 && operationMetadata.ld_countdown.get() == 0; } public void setConditionSources(String[] condition_sourceTable, String[] condition_source) { @@ -172,9 +143,9 @@ public void setConditionSources(String[] condition_sourceTable, String[] conditi public void initialize() { fd_parents.addAll(fd_concurrent_parents); fd_children.addAll(fd_concurrent_children); - operationMetadata.fd_countdown = fd_parents.size(); - operationMetadata.td_countdown = td_parents.size(); - operationMetadata.ld_countdown = ld_parents.size(); + operationMetadata.fd_countdown.set(fd_parents.size()); + operationMetadata.td_countdown.set(td_parents.size()); + operationMetadata.ld_countdown.set(ld_parents.size()); } public void addParent(Operation operation, DependencyType type) { @@ -245,25 +216,25 @@ public void updateDependencies(DependencyType dependencyType, OperationStateType // update countdown if (parentState.equals(OperationStateType.EXECUTED)) { if (dependencyType.equals(DependencyType.TD)) { - operationMetadata.td_countdown--; - assert operationMetadata.td_countdown >= 0; + operationMetadata.td_countdown.decrementAndGet(); + assert operationMetadata.td_countdown.get() >= 0; } else if (dependencyType.equals(DependencyType.FD)) { - operationMetadata.fd_countdown--; - assert operationMetadata.fd_countdown >= 0; + operationMetadata.fd_countdown.decrementAndGet(); + assert operationMetadata.fd_countdown.get() >= 0; } else if (dependencyType.equals(DependencyType.LD)) { - operationMetadata.ld_countdown--; - assert operationMetadata.ld_countdown >= 0; + operationMetadata.ld_countdown.decrementAndGet(); + assert operationMetadata.ld_countdown.get() >= 0; } } else if (parentState.equals(OperationStateType.READY) || parentState.equals(OperationStateType.BLOCKED)) { // rollback if (dependencyType.equals(DependencyType.TD)) { - operationMetadata.td_countdown++; - assert operationMetadata.td_countdown >= 0; + operationMetadata.td_countdown.incrementAndGet(); + assert operationMetadata.td_countdown.get() >= 0; } else if (dependencyType.equals(DependencyType.FD)) { - operationMetadata.fd_countdown++; - assert operationMetadata.fd_countdown >= 0; + operationMetadata.fd_countdown.incrementAndGet(); + assert operationMetadata.fd_countdown.get() >= 0; } else if (dependencyType.equals(DependencyType.LD)) { - operationMetadata.ld_countdown++; - assert operationMetadata.ld_countdown >= 0; + operationMetadata.ld_countdown.incrementAndGet(); + assert operationMetadata.ld_countdown.get() >= 0; } } else if (parentState.equals(OperationStateType.ABORTED)) { if (dependencyType.equals(DependencyType.TD)) { @@ -274,8 +245,8 @@ public void updateDependencies(DependencyType dependencyType, OperationStateType // operationMetadata.td_countdown--; // } // } - operationMetadata.td_countdown--; - assert operationMetadata.td_countdown >= 0; + operationMetadata.td_countdown.decrementAndGet(); + assert operationMetadata.td_countdown.get() >= 0; } else if (dependencyType.equals(DependencyType.FD)) { // operationMetadata.fd_countdown.set(fd_parents.size()); // for (Operation operation : fd_parents) { @@ -284,8 +255,8 @@ public void updateDependencies(DependencyType dependencyType, OperationStateType // operationMetadata.fd_countdown--; // } // } - operationMetadata.fd_countdown--; - assert operationMetadata.fd_countdown >= 0; + operationMetadata.fd_countdown.decrementAndGet(); + assert operationMetadata.fd_countdown.get() >= 0; } else if (dependencyType.equals(DependencyType.LD)) { // operationMetadata.ld_countdown.set(ld_parents.size()); // for (Operation operation : ld_parents) { @@ -294,8 +265,8 @@ public void updateDependencies(DependencyType dependencyType, OperationStateType // operationMetadata.ld_countdown--; // } // } - operationMetadata.ld_countdown--; - assert operationMetadata.ld_countdown >= 0; + operationMetadata.ld_countdown.decrementAndGet(); + assert operationMetadata.ld_countdown.get() >= 0; } } else { throw new UnsupportedOperationException(); @@ -303,16 +274,16 @@ public void updateDependencies(DependencyType dependencyType, OperationStateType } public void resetDependencies() { - operationMetadata.td_countdown = td_parents.size(); - operationMetadata.fd_countdown = fd_parents.size(); - operationMetadata.ld_countdown = ld_parents.size(); + operationMetadata.td_countdown.set(td_parents.size()); + operationMetadata.fd_countdown.set(fd_parents.size()); + operationMetadata.ld_countdown.set(ld_parents.size()); } public boolean tryReady() { - boolean isReady = operationMetadata.td_countdown == 0 - && operationMetadata.fd_countdown == 0 - && operationMetadata.ld_countdown == 0; + boolean isReady = operationMetadata.td_countdown.get() == 0 + && operationMetadata.fd_countdown.get() == 0 + && operationMetadata.ld_countdown.get() == 0; if (isReady) { stateTransition(OperationStateType.READY); @@ -323,7 +294,7 @@ public boolean tryReady() { // **********************************Utilities********************************** public boolean hasParents() { - return !(operationMetadata.td_countdown == 0 && operationMetadata.fd_countdown == 0 && operationMetadata.ld_countdown == 0); + return !(operationMetadata.td_countdown.get() == 0 && operationMetadata.fd_countdown.get() == 0 && operationMetadata.ld_countdown.get() == 0); } /** @@ -439,9 +410,9 @@ public void calculateDependencyLevelDuringExploration() { } } isDependencyLevelCalculated = true; - operationMetadata.td_countdown = td_parents.size(); - operationMetadata.fd_countdown = fd_parents.size(); - operationMetadata.ld_countdown = ld_parents.size(); + operationMetadata.td_countdown.set(td_parents.size()); + operationMetadata.fd_countdown.set(fd_parents.size()); + operationMetadata.ld_countdown.set(ld_parents.size()); } public void updateDependencyLevel(int dependencyLevel) { @@ -464,13 +435,14 @@ public boolean hasChildren() { private static class OperationMetadata { // countdown to be executed parents and committed parents for state transition -// public final AtomicInteger fd_countdown; -// public final AtomicInteger td_countdown; -// public final AtomicInteger ld_countdown; + // Has to use atomic for DFS, for notification from other threads, otherwise we may employ a state manager for DFS. + public final AtomicInteger fd_countdown; + public final AtomicInteger td_countdown; + public final AtomicInteger ld_countdown; - public int fd_countdown; - public int td_countdown; - public int ld_countdown; +// public int fd_countdown; +// public int td_countdown; +// public int ld_countdown; /** * A flag to control the state transition from a BLOCKED operation @@ -480,12 +452,12 @@ private static class OperationMetadata { * 2 can promote to a ready */ public OperationMetadata() { -// td_countdown = new AtomicInteger(0); // countdown Idx0/1 when executed/committed -// fd_countdown = new AtomicInteger(0); // countdown when executed -// ld_countdown = new AtomicInteger(0); // countdown when executed - td_countdown = 0; // countdown Idx0/1 when executed/committed - fd_countdown = 0; // countdown when executed - ld_countdown = 0; // countdown when executed + td_countdown = new AtomicInteger(0); // countdown Idx0/1 when executed/committed + fd_countdown = new AtomicInteger(0); // countdown when executed + ld_countdown = new AtomicInteger(0); // countdown when executed +// td_countdown = 0; // countdown Idx0/1 when executed/committed +// fd_countdown = 0; // countdown when executed +// ld_countdown = 0; // countdown when executed } } } \ No newline at end of file diff --git a/state-engine/src/main/java/transaction/impl/TxnManagerDedicatedAsy.java b/state-engine/src/main/java/transaction/impl/TxnManagerDedicatedAsy.java index 91ac28651..d3dd6311d 100644 --- a/state-engine/src/main/java/transaction/impl/TxnManagerDedicatedAsy.java +++ b/state-engine/src/main/java/transaction/impl/TxnManagerDedicatedAsy.java @@ -55,19 +55,13 @@ public void setSchedulerContext(int thisTaskId, int thread_count,String schedule SCHEDULER_TYPE scheduler_type = SCHEDULER_TYPE.valueOf(schedulerType); switch (scheduler_type) { case OG_BFS: - context = new OGBFSContext(thisTaskId, thread_count); - scheduler.AddContext(thisTaskId, context); - break; - case OG_BFS_A: - context = new OGBFSAContext(thisTaskId, thread_count); - scheduler.AddContext(thisTaskId, context); - break; case OG_DFS: - context = new OGDFSContext(thisTaskId, thread_count); + context = new OGSContext(thisTaskId, thread_count); scheduler.AddContext(thisTaskId, context); break; + case OG_BFS_A: case OG_DFS_A: - context = new OGDFSAContext(thisTaskId, thread_count); + context = new OGSAContext(thisTaskId, thread_count); scheduler.AddContext(thisTaskId, context); break; case OG_NS: From 2aedfbe561b40a6535c1304668a02a2fa4d77a2c Mon Sep 17 00:00:00 2001 From: curryzjj Date: Mon, 14 Mar 2022 16:04:52 +0800 Subject: [PATCH 08/15] solve conflict --- common/src/main/java/common/Runner.java | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/common/src/main/java/common/Runner.java b/common/src/main/java/common/Runner.java index 7f35ffd57..8f00b58d2 100644 --- a/common/src/main/java/common/Runner.java +++ b/common/src/main/java/common/Runner.java @@ -57,7 +57,7 @@ public abstract class Runner implements IRunner { * TStream Specific Parameters. */ @Parameter(names = {"--tthread"}, description = "total execution threads") - public int tthread = 4;// default total execution threads + public int tthread = 1;// default total execution threads @Parameter(names = {"--CCOption"}, description = "Selecting different concurrency control options.") public int CCOption = CCOption_MorphStream; // public int CCOption = CCOption_SStore; @@ -65,14 +65,14 @@ public abstract class Runner implements IRunner { @Parameter(names = {"--partition"}, description = "Partitioning database. It must be enabled for S-Store scheme and it is optional for TStream scheme.") public boolean enable_partition = false; @Parameter(names = {"--scheduler"}, description = "Scheduler for TStream.") - // public String scheduler = "OG_DFS" + public String scheduler = "OG_BFS"; // public String scheduler = "OG_BFS_A"; // public String scheduler = "OG_DFS"; // public String scheduler = "OG_DFS_A"; // public String scheduler = "OG_NS"; // public String scheduler = "OG_NS_A"; // public String scheduler = "OP_NS"; - public String scheduler = "OP_NS_A"; +// public String scheduler = "OP_NS_A"; // public String scheduler = "OP_BFS"; // public String scheduler = "OP_BFS_A"; // public String scheduler = "OP_DFS"; @@ -86,18 +86,12 @@ public abstract class Runner implements IRunner { /** * Dynamic Scheduler */ - @Parameter(names = {"--isRuntime"}, description = "Collect runtime information") - public boolean isRuntime = false; @Parameter(names = {"--isDynamic"}, description = "Dynamic Workload") public boolean isDynamic = false; @Parameter(names = {"--schedulerPool"}, description = "Schedulers in the SchedulerPool[OG_DFS,OP_DFS]") - public String schedulerPools = "OP_NS_A,OP_BFS_A,OP_BFS,OG_BFS"; + public String schedulerPools = "OG_DFS,OP_DFS"; @Parameter(names = {"--defaultScheduler"}, description = "Default scheduler") public String defaultScheduler = "OG_DFS"; - @Parameter(names = {"--bottomLine"}, description = "BottomLine for(TD,LD,PD,SUM,VDD,R_of_A)") - public String bottomLine = "2,3,5,10,0.6,0.7"; - @Parameter(names = {"--WorkloadConfig"}, description = "WorkloadConfigs(TD,LD,PD,VDD,R_of_A,isCD,isCC,markId)") - public String WorkloadConfig = "1,3,4,0.5,0.6,1,1,39999;1,6,4,0.5,0.7,1,1,79999;3,6,3,0.5,0.7,1,1,100000"; /** * Benchmarking Specific Parameters. @@ -121,7 +115,7 @@ public abstract class Runner implements IRunner { * generator parameters */ @Parameter(names = {"--checkpoint_interval"}, description = "checkpoint interval (#tuples)") - public int checkpoint_interval = 10000;//checkpoint per thread. + public int checkpoint_interval = 20000;//checkpoint per thread. @Parameter(names = {"--generator"}, description = "Generator for TStream.") public String generator = "TPGGenerator"; // public String generator = "OCGenerator"; @@ -225,11 +219,7 @@ public void initializeCfg(HashMap config) { /* Dynamic Workload Configuration*/ config.put("isDynamic",isDynamic); config.put("schedulersPool",schedulerPools); - config.put("defaultScheduler",scheduler); - config.put("isRuntime",isRuntime); - config.put("bottomLine",bottomLine); - config.put("WorkloadConfig",WorkloadConfig); - + config.put("defaultScheduler",defaultScheduler); System.setProperty("my.log", metric_path); } } From 38c100260c873bc9e91f531e3444d53c22a22d55 Mon Sep 17 00:00:00 2001 From: curryzjj Date: Tue, 15 Mar 2022 10:53:46 +0800 Subject: [PATCH 09/15] Solve the conflict --- affinity/src/main/c/Makefile | 6 ++--- common/src/main/java/common/Runner.java | 22 ++++++++++++++----- .../java/scheduler/impl/og/OGScheduler.java | 4 ++++ .../java/scheduler/impl/op/OPScheduler.java | 2 +- .../struct/og/TaskPrecedenceGraph.java | 4 ++-- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/affinity/src/main/c/Makefile b/affinity/src/main/c/Makefile index 85e114c74..6bb98b2a4 100755 --- a/affinity/src/main/c/Makefile +++ b/affinity/src/main/c/Makefile @@ -19,8 +19,8 @@ JNI_JAVA_SOURCES := $(patsubst %,$(TARGET_DIR)/%.class,$(subst .,/,$(JAVA_CLASSE JAVA_BUILD_DIR := $(TARGET_DIR) -#JAVA_HOME ?= /usr/java/default -JAVA_HOME :=/Library/Java/JavaVirtualMachines/jdk1.8.0_191.jdk/Contents/Home +JAVA_HOME ?= /usr/java/default +#JAVA_HOME :=/Library/Java/JavaVirtualMachines/jdk1.8.0_191.jdk/Contents/Home JAVA_LIB := $(JAVA_HOME)/jre/lib JVM_SHARED_LIBS := -L$(JAVA_LIB)/amd64/server -L$(JAVA_LIB)/i386/server -L$(JAVA_LIB)/amd64/jrockit/ -L$(JAVA_LIB)/i386/jrockit/ -L$(JAVA_LIB)/ppc64le/server -L$(JAVA_LIB)/ppc64le/jrockit/ @@ -43,7 +43,7 @@ $(TARGET): $(JNI_HEADERS) $(JNI_SOURCES) $(JNI_HEADERS): $(JNI_JAVA_SOURCES) mkdir -p $(TARGET_DIR)/jni - javac -h -force -classpath $(JAVAH_CLASSPATH) -d $(WORKING_DIR) $(JAVA_CLASSES) + javah -force -classpath $(JAVAH_CLASSPATH) -d $(WORKING_DIR) $(JAVA_CLASSES) clean: rm $(TARGET) diff --git a/common/src/main/java/common/Runner.java b/common/src/main/java/common/Runner.java index 8f00b58d2..2f406c450 100644 --- a/common/src/main/java/common/Runner.java +++ b/common/src/main/java/common/Runner.java @@ -57,7 +57,7 @@ public abstract class Runner implements IRunner { * TStream Specific Parameters. */ @Parameter(names = {"--tthread"}, description = "total execution threads") - public int tthread = 1;// default total execution threads + public int tthread = 4;// default total execution threads @Parameter(names = {"--CCOption"}, description = "Selecting different concurrency control options.") public int CCOption = CCOption_MorphStream; // public int CCOption = CCOption_SStore; @@ -86,12 +86,18 @@ public abstract class Runner implements IRunner { /** * Dynamic Scheduler */ + @Parameter(names = {"--isRuntime"}, description = "Collect runtime information") + public boolean isRuntime = false; @Parameter(names = {"--isDynamic"}, description = "Dynamic Workload") - public boolean isDynamic = false; + public boolean isDynamic = true; @Parameter(names = {"--schedulerPool"}, description = "Schedulers in the SchedulerPool[OG_DFS,OP_DFS]") - public String schedulerPools = "OG_DFS,OP_DFS"; + public String schedulerPools = "OP_NS_A,OP_BFS_A,OP_BFS,OG_BFS"; @Parameter(names = {"--defaultScheduler"}, description = "Default scheduler") - public String defaultScheduler = "OG_DFS"; + public String defaultScheduler = "OG_BFS"; + @Parameter(names = {"--bottomLine"}, description = "BottomLine for(TD,LD,PD,SUM,VDD,R_of_A)") + public String bottomLine = "2,3,5,10,0.6,0.7"; + @Parameter(names = {"--WorkloadConfig"}, description = "WorkloadConfigs(TD,LD,PD,VDD,R_of_A,isCD,isCC,markId)") + public String WorkloadConfig = "1,3,4,0.5,0.6,1,1,39999;1,6,4,0.5,0.7,1,1,79999;3,6,3,0.5,0.7,1,1,100000"; /** * Benchmarking Specific Parameters. @@ -115,7 +121,7 @@ public abstract class Runner implements IRunner { * generator parameters */ @Parameter(names = {"--checkpoint_interval"}, description = "checkpoint interval (#tuples)") - public int checkpoint_interval = 20000;//checkpoint per thread. + public int checkpoint_interval = 2500;//checkpoint per thread. @Parameter(names = {"--generator"}, description = "Generator for TStream.") public String generator = "TPGGenerator"; // public String generator = "OCGenerator"; @@ -219,7 +225,11 @@ public void initializeCfg(HashMap config) { /* Dynamic Workload Configuration*/ config.put("isDynamic",isDynamic); config.put("schedulersPool",schedulerPools); - config.put("defaultScheduler",defaultScheduler); + config.put("defaultScheduler",scheduler); + config.put("isRuntime",isRuntime); + config.put("bottomLine",bottomLine); + config.put("WorkloadConfig",WorkloadConfig); + System.setProperty("my.log", metric_path); } } diff --git a/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java b/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java index 303e77c0b..cb268da32 100644 --- a/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java @@ -357,6 +357,10 @@ private Operation constructOp(List operationGraph, Request request) { set_op = new Operation(request.src_key, request.function, request.table_name, request.record_ref, request.condition_records, request.condition, request.success, request.txn_context, request.accessType, request.d_record, request.d_record, bid, targetContext); break; + case READ_WRITE_READ: + set_op = new Operation(request.src_key, request.function, request.table_name, request.record_ref, null, null, + null, request.txn_context, request.accessType, request.d_record, request.d_record, bid, targetContext); + break; default: throw new RuntimeException("Unexpected operation"); } diff --git a/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java b/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java index 9ef91de5c..be0394d95 100644 --- a/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java @@ -248,7 +248,7 @@ public void TxnSubmitFinished(Context context) { break; case READ_WRITE_READ: set_op = new Operation(request.src_key, getTargetContext(request.src_key), request.table_name, request.txn_context, bid, request.accessType, - request.d_record, request.record_ref, request.function); + request.d_record, request.record_ref, request.function, null, null, null); break; default: throw new UnsupportedOperationException(); diff --git a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java index 201ba3abb..2c55949bd 100644 --- a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java @@ -118,8 +118,8 @@ public void setOCs(Context context) { ocs.add(accOC); ocs.add(beOC); }else if(app==2){ - SchedulingUnit speedOC=context.createTask("segment_speed",_key,0); - SchedulingUnit cntOC=context.createTask("segment_cnt",_key,0); + OperationChain speedOC=context.createTask("segment_speed",_key,0); + OperationChain cntOC=context.createTask("segment_cnt",_key,0); operationChains.get("segment_speed").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, speedOC); operationChains.get("segment_cnt").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, cntOC); ocs.add(speedOC); From 4678f243e55d6c08d7180e6b794fb3a084eb001c Mon Sep 17 00:00:00 2001 From: curryzjj Date: Wed, 16 Mar 2022 20:40:43 +0800 Subject: [PATCH 10/15] Add DynamicWorkloadGenerator --- .../datagenerator/DataGenerator.java | 8 + .../TPGTxnGenerator/GSTPGDataGenerator.java | 1 + .../GSTPGDataGeneratorConfig.java | 8 + .../GSTPGDynamicDataGenerator.java | 236 ++++++++++++++ .../SLTPGDynamicDataGenerator.java | 298 ++++++++++++++++++ .../DynamicDataGeneratorConfig.java | 67 ++++ .../DynamicWorkloadGenerator.java | 41 +++ .../initializer/GSInitializer.java | 44 ++- .../initializer/SLInitializer.java | 28 +- common/src/main/java/common/Runner.java | 20 +- .../java/scheduler/collector/Collector.java | 2 +- .../main/java/execution/ExecutionManager.java | 1 - 12 files changed, 730 insertions(+), 24 deletions(-) create mode 100644 application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java create mode 100644 application/src/main/java/benchmark/datagenerator/apps/SL/TPGTxnGenerator/SLTPGDynamicDataGenerator.java create mode 100644 application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java create mode 100644 application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java diff --git a/application/src/main/java/benchmark/datagenerator/DataGenerator.java b/application/src/main/java/benchmark/datagenerator/DataGenerator.java index 56c353333..76590e86b 100644 --- a/application/src/main/java/benchmark/datagenerator/DataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/DataGenerator.java @@ -2,6 +2,7 @@ import benchmark.datagenerator.apps.SL.output.GephiOutputHandler; import benchmark.datagenerator.apps.SL.output.IOutputHandler; +import benchmark.dynamicWorkloadGenerator.DynamicDataGeneratorConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,6 +21,11 @@ public DataGenerator(DataGeneratorConfig dataConfig) { this.dataOutputHandler = new GephiOutputHandler(dataConfig.getRootPath()); } + public DataGenerator(DynamicDataGeneratorConfig dynamicDataConfig) { + this.nTuples = dynamicDataConfig.getTotalEvents(); + this.dataOutputHandler = new GephiOutputHandler(dynamicDataConfig.getRootPath()); + } + public T getDataConfig() { return (T) dataConfig; } @@ -37,5 +43,7 @@ public void generateStream() { public void clearDataStructures() { this.dataConfig = null; } + /* Switch configuration, used in the dynamic data generator*/ + } diff --git a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java index e619e35a3..d5cb67fdb 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java @@ -1,6 +1,7 @@ package benchmark.datagenerator.apps.GS.TPGTxnGenerator; import benchmark.datagenerator.DataGenerator; +import benchmark.datagenerator.DataGeneratorConfig; import benchmark.datagenerator.Event; import benchmark.datagenerator.apps.GS.TPGTxnGenerator.Transaction.GSEvent; import common.tools.FastZipfGenerator; diff --git a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java index d716a0dde..e2d6c6963 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java +++ b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java @@ -26,4 +26,12 @@ public void initialize(Configuration config) { Ratio_of_Transaction_Aborts = config.getInt("Ratio_of_Transaction_Aborts", 0); Transaction_Length = config.getInt("Transaction_Length", 1); } + + public void setConfiguration(int num_access, int state_Access_Skewness, int ratio_of_Overlapped_Keys,int ratio_of_Transaction_Aborts, int transaction_Length){ + NUM_ACCESS = num_access; + State_Access_Skewness = state_Access_Skewness; + Ratio_of_Overlapped_Keys = ratio_of_Overlapped_Keys; + Ratio_of_Transaction_Aborts = ratio_of_Transaction_Aborts; + Transaction_Length = transaction_Length; + } } \ No newline at end of file diff --git a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java new file mode 100644 index 000000000..91c08cbfa --- /dev/null +++ b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java @@ -0,0 +1,236 @@ +package benchmark.datagenerator.apps.GS.TPGTxnGenerator; +import benchmark.datagenerator.DataGeneratorConfig; +import benchmark.datagenerator.Event; +import benchmark.datagenerator.apps.GS.TPGTxnGenerator.Transaction.GSEvent; +import benchmark.dynamicWorkloadGenerator.DynamicDataGeneratorConfig; +import benchmark.dynamicWorkloadGenerator.DynamicWorkloadGenerator; +import common.tools.FastZipfGenerator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import utils.AppConfig; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Random; + +import static common.CONTROL.enable_log; +import static common.CONTROL.enable_states_partition; + +public class GSTPGDynamicDataGenerator extends DynamicWorkloadGenerator { + private static final Logger LOG = LoggerFactory.getLogger(GSTPGDynamicDataGenerator.class); + private int NUM_ACCESS; // transaction length, 4 or 8 or longer + private int State_Access_Skewness; // ratio of state access, following zipf distribution + private int Ratio_of_Transaction_Aborts; // ratio of transaction aborts, fail the transaction or not. i.e. transfer amount might be invalid. + private int Ratio_of_Overlapped_Keys; // ratio of overlapped keys in transactions, which affects the dependencies and circulars. + private int Transaction_Length; + private int tthread; + // control the number of txns overlap with each other. + private ArrayList generatedKeys = new ArrayList<>(); + // independent transactions. + private boolean isUnique = false; + private FastZipfGenerator keyZipf; + + private int floor_interval; + public FastZipfGenerator[] partitionedKeyZipf; + + private Random random = new Random(0); // the transaction type decider + public transient FastZipfGenerator p_generator; // partition generator + private HashMap nGeneratedIds = new HashMap<>(); + private ArrayList events; + private int eventID = 0; + private HashMap idToLevel = new HashMap<>(); + + public GSTPGDynamicDataGenerator(DynamicDataGeneratorConfig dynamicDataConfig) { + super(dynamicDataConfig); + events = new ArrayList<>(nTuples); + } + + @Override + public void switchConfiguration() { + State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; + NUM_ACCESS = dynamicDataConfig.NUM_ACCESS; + Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; + Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; + Transaction_Length = dynamicDataConfig.Transaction_Length; + + int nKeyState = dynamicDataConfig.getnKeyStates(); + int MAX_LEVEL = 256; + for (int i = 0; i < nKeyState; i++) { + idToLevel.put(i, i% MAX_LEVEL); + } + keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); + configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); + p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); + } + + @Override + protected void generateTuple() { + GSEvent event; + event = randomEvent(); + events.add(event); + } + + @Override + public void dumpGeneratedDataToFile() { + if (enable_log) LOG.info("++++++" + nGeneratedIds.size()); + + if (enable_log) LOG.info("Dumping transactions..."); + try { + dataOutputHandler.sinkEvents(events); + } catch (IOException e) { + e.printStackTrace(); + } + + File versionFile = new File(dynamicDataConfig.getRootPath().substring(0, dynamicDataConfig.getRootPath().length() - 1) + + String.format("_%d.txt", dynamicDataConfig.getTotalEvents())); + try { + versionFile.createNewFile(); + FileWriter fileWriter = new FileWriter(versionFile); + fileWriter.write(String.format("Total number of threads : %d\n", dynamicDataConfig.getTotalThreads())); + fileWriter.write(String.format("Total Events : %d\n", dynamicDataConfig.getTotalEvents())); + fileWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void configure_store(double scale_factor, double theta, int tthread, int numItems) { + floor_interval = (int) Math.floor(numItems / (double) tthread);//NUM_ITEMS / tthread; + partitionedKeyZipf = new FastZipfGenerator[tthread];//overhead_total number of working threads. + for (int i = 0; i < tthread; i++) { + partitionedKeyZipf[i] = new FastZipfGenerator((int) (floor_interval * scale_factor), theta, i * floor_interval, 12345678); + } + } + + private GSEvent randomEvent() { + int[] keys = new int[NUM_ACCESS*Transaction_Length]; + int writeLevel = -1; + if (!isUnique) { + if (enable_states_partition) { + for (int j = 0; j < Transaction_Length; j++) { + int partitionId = key_to_partition(p_generator.next()); + for (int i = 0; i < NUM_ACCESS; i++) { + int offset = j * NUM_ACCESS + i; + if (AppConfig.isCyclic) { + int key = getKey(partitionedKeyZipf[partitionId], partitionId, generatedKeys); + if (offset % NUM_ACCESS == 0) { + // make sure this one is different with other write key + for (int k = 0; k < j; k++) { + while (keys[k*NUM_ACCESS] == key) { + key = getKey(partitionedKeyZipf[partitionId], partitionId, generatedKeys); + } + } + } + keys[offset] = key; + } else { + // TODO: correct it later + keys[offset] = getKey(partitionedKeyZipf[partitionId], partitionId, generatedKeys); + if (i == 0) { + while (idToLevel.get(keys[offset]) == 0) { + keys[offset] = getKey(partitionedKeyZipf[partitionId], partitionId, generatedKeys); + } + writeLevel = idToLevel.get(keys[offset]); + } else { + while (writeLevel <= idToLevel.get(keys[offset])) { + keys[offset] = getKey(partitionedKeyZipf[partitionId], partitionId, generatedKeys); + } + } + } + partitionId = (partitionId + 1) % dynamicDataConfig.getTotalThreads(); + } + } + } else { + for (int i = 0; i < NUM_ACCESS; i++) { + if (AppConfig.isCyclic) { + keys[i] = getKey(keyZipf, generatedKeys); + } else { + keys[i] = getKey(keyZipf, generatedKeys); + if (i == 0) { + while (idToLevel.get(keys[i]) == 0) { + keys[i] = getKey(keyZipf, generatedKeys); + } + writeLevel = idToLevel.get(keys[i]); + } + while (writeLevel <= idToLevel.get(keys[i])) { + keys[i] = getKey(keyZipf, generatedKeys); + } + } + } + } + } else { + // TODO: add transaction length logic + for (int i = 0; i generatedKeys) { + int key; + key = zipfGenerator.next(); + int next = random.nextInt(100); + if (next < Ratio_of_Overlapped_Keys) { // randomly select a key from existing keyset. + if (!generatedKeys.isEmpty()) { + int counter = 0; + key = generatedKeys.get(random.nextInt(generatedKeys.size())); + while (key / floor_interval != partitionId) { + key = generatedKeys.get(random.nextInt(generatedKeys.size())); + counter++; + if (counter >= partitionId) { + key = zipfGenerator.next(); + break; + } + } + } + } + return key; + } + + private int getKey(FastZipfGenerator zipfGenerator, ArrayList generatedKeys) { + int srcKey; + srcKey = zipfGenerator.next(); + int next = random.nextInt(100); + if (next < Ratio_of_Overlapped_Keys) { // randomly select a key from existing keyset. + if (!generatedKeys.isEmpty()) { + srcKey = generatedKeys.get(zipfGenerator.next() % generatedKeys.size()); + } + } + return srcKey; + } + + private int getUniqueKey(FastZipfGenerator zipfGenerator, ArrayList generatedKeys) { + int key; + key = zipfGenerator.next(); + while (generatedKeys.contains(key)) { + key = zipfGenerator.next(); + } + generatedKeys.add(key); + return key; + } + + + +} diff --git a/application/src/main/java/benchmark/datagenerator/apps/SL/TPGTxnGenerator/SLTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/SL/TPGTxnGenerator/SLTPGDynamicDataGenerator.java new file mode 100644 index 000000000..34853e996 --- /dev/null +++ b/application/src/main/java/benchmark/datagenerator/apps/SL/TPGTxnGenerator/SLTPGDynamicDataGenerator.java @@ -0,0 +1,298 @@ +package benchmark.datagenerator.apps.SL.TPGTxnGenerator; + +import benchmark.datagenerator.Event; +import benchmark.datagenerator.apps.SL.Transaction.SLDepositEvent; +import benchmark.datagenerator.apps.SL.Transaction.SLTransferEvent; +import benchmark.dynamicWorkloadGenerator.DynamicDataGeneratorConfig; +import benchmark.dynamicWorkloadGenerator.DynamicWorkloadGenerator; +import common.tools.FastZipfGenerator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import utils.AppConfig; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Random; + +import static common.CONTROL.enable_log; +import static common.CONTROL.enable_states_partition; + +public class SLTPGDynamicDataGenerator extends DynamicWorkloadGenerator { + private static final Logger LOG = LoggerFactory.getLogger(SLTPGDynamicDataGenerator.class); + + private int Ratio_Of_Deposit; // ratio of state access type i.e. deposit or transfer + private int State_Access_Skewness; // ratio of state access, following zipf distribution + private int Transaction_Length; // transaction length, 4 or 8 or longer + private int Ratio_of_Transaction_Aborts; // ratio of transaction aborts, fail the transaction or not. i.e. transfer amount might be invalid. + private int Ratio_of_Overlapped_Keys; // ratio of overlapped keys in transactions, which affects the dependencies and circulars. + // control the number of txns overlap with each other. + private ArrayList generatedAcc = new ArrayList<>(); + private ArrayList generatedAst = new ArrayList<>(); + // independent transactions. + private final boolean isUnique = false; + private FastZipfGenerator accountZipf; + private FastZipfGenerator assetZipf; + + private int floor_interval; + public FastZipfGenerator[] partitionedAccountZipf; + public FastZipfGenerator[] partitionedAssetZipf; + + + private Random random = new Random(0); // the transaction type decider + public transient FastZipfGenerator p_generator; // partition generator + private HashMap nGeneratedAccountIds = new HashMap<>(); + private HashMap nGeneratedAssetIds = new HashMap<>(); + private ArrayList events; + private int eventID = 0; + private HashMap idToLevel = new HashMap<>(); + public SLTPGDynamicDataGenerator(DynamicDataGeneratorConfig dynamicDataConfig){ + super(dynamicDataConfig); + events = new ArrayList<>(nTuples); + } + @Override + public void switchConfiguration() { + Ratio_Of_Deposit = dynamicDataConfig.Ratio_Of_Deposit;//0-100 (%) + State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; + Transaction_Length = 4; + Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; + Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; + + int nKeyState = dynamicDataConfig.getnKeyStates(); + + // allocate levels for each key, to prevent circular. +// int MAX_LEVEL = (nKeyState / dataConfig.getTotalThreads()) / 2; + int MAX_LEVEL = 256; + for (int i = 0; i < nKeyState; i++) { + idToLevel.put(i, random.nextInt(MAX_LEVEL)); + } + // zipf state access generator + accountZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); + assetZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 123456789); + configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); + p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); + + } + public void configure_store(double scale_factor, double theta, int tthread, int numItems) { + floor_interval = (int) Math.floor(numItems / (double) tthread);//NUM_ITEMS / tthread; + partitionedAccountZipf = new FastZipfGenerator[tthread];//overhead_total number of working threads. + partitionedAssetZipf = new FastZipfGenerator[tthread];//overhead_total number of working threads. + for (int i = 0; i < tthread; i++) { + partitionedAccountZipf[i] = new FastZipfGenerator((int) (floor_interval * scale_factor), theta, i * floor_interval, 12345678); + partitionedAssetZipf[i] = new FastZipfGenerator((int) (floor_interval * scale_factor), theta, i * floor_interval, 123456789); + } + } + @Override + protected void generateTuple() { + Event event; + int next = random.nextInt(100); + if (next < Ratio_Of_Deposit) { + event = randomDepositEvent(); + } else { + event = randomTransferEvent(); + } + events.add(event); + } + @Override + public void dumpGeneratedDataToFile() { + if (enable_log) LOG.info("++++++" + nGeneratedAccountIds.size()); + if (enable_log) LOG.info("++++++" + nGeneratedAssetIds.size()); + + if (enable_log) LOG.info("Dumping transactions..."); + try { + dataOutputHandler.sinkEvents(events); + } catch (IOException e) { + e.printStackTrace(); + } + + File versionFile = new File(dynamicDataConfig.getRootPath().substring(0, dynamicDataConfig.getRootPath().length() - 1) + + String.format("_%d.txt", dynamicDataConfig.getTotalEvents())); + try { + versionFile.createNewFile(); + FileWriter fileWriter = new FileWriter(versionFile); + fileWriter.write(String.format("Total number of threads : %d\n", dynamicDataConfig.getTotalThreads())); + fileWriter.write(String.format("Total Events : %d\n", dynamicDataConfig.getTotalEvents())); + fileWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + private Event randomTransferEvent() { + // make sure source and destination are different + int srcAcc, dstAcc, srcAst, dstAst; + + if (!isUnique) { + if (enable_states_partition) { + int partitionId = key_to_partition(p_generator.next()); + int[] accKeys = getKeys(partitionedAccountZipf[partitionId], + partitionedAccountZipf[(partitionId+2) % dynamicDataConfig.getTotalThreads()], + partitionId, + (partitionId+2) % dynamicDataConfig.getTotalThreads(), + generatedAcc); + srcAcc = accKeys[0]; + dstAcc = accKeys[1]; + int[] astKeys = getKeys(partitionedAssetZipf[(partitionId+1) % dynamicDataConfig.getTotalThreads()], + partitionedAssetZipf[(partitionId+3) % dynamicDataConfig.getTotalThreads()], + (partitionId+1) % dynamicDataConfig.getTotalThreads(), + (partitionId+3) % dynamicDataConfig.getTotalThreads(), + generatedAst); + srcAst = astKeys[0]; + dstAst = astKeys[1]; + assert srcAcc / floor_interval == partitionId; + assert srcAst / floor_interval == (partitionId+1) % dynamicDataConfig.getTotalThreads(); + assert dstAcc / floor_interval == (partitionId+2) % dynamicDataConfig.getTotalThreads(); + assert dstAst / floor_interval == (partitionId+3) % dynamicDataConfig.getTotalThreads(); + } else { + int[] accKeys = getKeys(accountZipf, generatedAcc); + srcAcc = accKeys[0]; + dstAcc = accKeys[1]; + int[] astKeys = getKeys(assetZipf, generatedAst); + srcAst = astKeys[0]; + dstAst = astKeys[1]; + } + } else { // generate unique keys for unique txn processing + srcAcc = getUniqueKey(accountZipf, generatedAcc); + dstAcc = getUniqueKey(accountZipf, generatedAcc); + srcAst = getUniqueKey(assetZipf, generatedAst); + dstAst = getUniqueKey(assetZipf, generatedAst); + } + + assert srcAcc != dstAcc; + assert srcAst != dstAst; + + // just for stats record + nGeneratedAccountIds.put(srcAcc, nGeneratedAccountIds.getOrDefault((long) srcAcc, 0) + 1); + nGeneratedAccountIds.put(dstAcc, nGeneratedAccountIds.getOrDefault((long) dstAcc, 0) + 1); + nGeneratedAssetIds.put(srcAst, nGeneratedAccountIds.getOrDefault((long) srcAst, 0) + 1); + nGeneratedAssetIds.put(dstAst, nGeneratedAccountIds.getOrDefault((long) dstAst, 0) + 1); + Event t; + if (random.nextInt(10000) < Ratio_of_Transaction_Aborts) { + t = new SLTransferEvent(eventID, srcAcc, srcAst, dstAcc, dstAst, 100000000, 100000000); + } else { + t = new SLTransferEvent(eventID, srcAcc, srcAst, dstAcc, dstAst); + } + + // increase the timestamp i.e. transaction id + eventID++; + return t; + } + + private Event randomDepositEvent() { + int acc; + int ast; + if(!isUnique) { + if (enable_states_partition) { + int partitionId = key_to_partition(p_generator.next()); + acc = getKey(partitionedAccountZipf[partitionId], partitionId, generatedAcc); + ast = getKey(partitionedAssetZipf[(partitionId+1) % dynamicDataConfig.getTotalThreads()], (partitionId+1) % dynamicDataConfig.getTotalThreads(), generatedAst); + } else { + acc = getKey(accountZipf, generatedAcc); + ast = getKey(assetZipf, generatedAst); + } + } else { + acc = getUniqueKey(accountZipf, generatedAcc); + ast = getUniqueKey(assetZipf, generatedAst); + } + + // just for stats record + nGeneratedAccountIds.put(acc, nGeneratedAccountIds.getOrDefault((long) acc, 0) + 1); + nGeneratedAssetIds.put(ast, nGeneratedAccountIds.getOrDefault((long) ast, 0) + 1); + + Event t = new SLDepositEvent(eventID, acc, ast); + + // increase the timestamp i.e. transaction id + eventID++; + return t; + } + public int key_to_partition(int key) { + return (int) Math.floor((double) key / floor_interval); + } + + private int[] getKeys(FastZipfGenerator zipfGeneratorSrc, FastZipfGenerator zipfGeneratorDst, int partitionSrc, int partitionDst, ArrayList generatedKeys) { + int[] keys = new int[2]; + keys[0] = getKey(zipfGeneratorSrc, partitionSrc, generatedKeys); + keys[1] = getKey(zipfGeneratorDst, partitionDst, generatedKeys); + + if (AppConfig.isCyclic) { // whether to generate acyclic input stream. + while (keys[0] == keys[1]) { + keys[0] = getKey(zipfGeneratorSrc, partitionSrc, generatedKeys); + keys[1] = getKey(zipfGeneratorDst, partitionDst, generatedKeys); + } + } else { + while (keys[0] == keys[1] || idToLevel.get(keys[0]) >= idToLevel.get(keys[1])) { + keys[0] = getKey(zipfGeneratorSrc, partitionSrc, generatedKeys); + keys[1] = getKey(zipfGeneratorDst, partitionDst, generatedKeys); + } + } + + generatedKeys.add(keys[0]); + generatedKeys.add(keys[1]); + return keys; + } + private int getKey(FastZipfGenerator zipfGenerator, int partitionId, ArrayList generatedKeys) { + int key; + key = zipfGenerator.next(); + int next = random.nextInt(100); + if (next < Ratio_of_Overlapped_Keys) { // randomly select a key from existing keyset. + if (!generatedKeys.isEmpty()) { + int counter = 0; + key = generatedKeys.get(random.nextInt(generatedKeys.size())); + while (key / floor_interval != partitionId) { + key = generatedKeys.get(random.nextInt(generatedKeys.size())); + counter++; + if (counter >= partitionId) { + key = zipfGenerator.next(); + break; + } + } + } + } + return key; + } + + private int[] getKeys(FastZipfGenerator zipfGenerator, ArrayList generatedKeys) { + int[] keys = new int[2]; + keys[0] = getKey(zipfGenerator, generatedKeys); + keys[1] = getKey(zipfGenerator, generatedKeys); + + if (AppConfig.isCyclic) { + while (keys[0] == keys[1]) { + keys[0] = getKey(zipfGenerator, generatedKeys); + keys[1] = getKey(zipfGenerator, generatedKeys); + } + } else { + while (keys[0] == keys[1] || idToLevel.get(keys[0]) >= idToLevel.get(keys[1])) { + keys[0] = getKey(zipfGenerator, generatedKeys); + keys[1] = getKey(zipfGenerator, generatedKeys); + } + } + + generatedKeys.add(keys[0]); + generatedKeys.add(keys[1]); + return keys; + } + + private int getKey(FastZipfGenerator zipfGenerator, ArrayList generatedKeys) { + int srcKey; + srcKey = zipfGenerator.next(); + int next = random.nextInt(100); + if (next < Ratio_of_Overlapped_Keys) { // randomly select a key from existing keyset. + if (!generatedKeys.isEmpty()) { + srcKey = generatedKeys.get(zipfGenerator.next() % generatedKeys.size()); + } + } + return srcKey; + } + + private int getUniqueKey(FastZipfGenerator zipfGenerator, ArrayList generatedKeys) { + int key; + key = zipfGenerator.next(); + while (generatedKeys.contains(key)) { + key = zipfGenerator.next(); + } + generatedKeys.add(key); + return key; + } +} diff --git a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java new file mode 100644 index 000000000..2f9680259 --- /dev/null +++ b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java @@ -0,0 +1,67 @@ +package benchmark.dynamicWorkloadGenerator; + +import benchmark.datagenerator.DataGeneratorConfig; +import benchmark.datagenerator.apps.GS.TPGTxnGenerator.GSTPGDataGeneratorConfig; +import benchmark.datagenerator.apps.SL.TPGTxnGenerator.SLTPGDataGeneratorConfig; +import common.collections.Configuration; + +import java.util.ArrayList; +import java.util.List; + +/** + * Generate dynamic workload configuration + * 1. workload configuration generation + * Created by curry on 16/3/22. + */ +public class DynamicDataGeneratorConfig extends DataGeneratorConfig { + /* The type of the dynamic workload */ + private String type; + /* Application, the configuration maybe different for the same type of workload */ + private String app; + /* Control the rate of the workload*/ + private int shiftRate; + private int checkpoint_interval; + public int State_Access_Skewness; + public int NUM_ACCESS; + public int Ratio_of_Overlapped_Keys; + public int Ratio_of_Transaction_Aborts; + public int Transaction_Length; + public int Ratio_Of_Deposit; + + public void initialize(Configuration config) { + super.initialize(config); + this.app = config.getString("application"); + this.type = config.getString("workloadType"); + this.shiftRate = config.getInt("shiftRate"); + this.checkpoint_interval = config.getInt("checkpoint"); + NUM_ACCESS = config.getInt("NUM_ACCESS", 0); + State_Access_Skewness = config.getInt("State_Access_Skewness", 0); + Ratio_of_Overlapped_Keys = config.getInt("Ratio_of_Overlapped_Keys", 0); + Ratio_of_Transaction_Aborts = config.getInt("Ratio_of_Transaction_Aborts", 0); + Transaction_Length = config.getInt("Transaction_Length", 1); + Ratio_Of_Deposit = config.getInt("Ratio_Of_Deposit", 0); + } + + public int getCheckpoint_interval() { + return checkpoint_interval; + } + + public int getShiftRate() { + return shiftRate; + } + /** + * Generate the configuration based on type and application + * @return + */ + public void nextDataGeneratorConfig() { + + } + + public String getType() { + return type; + } + + public String getApp() { + return app; + } +} diff --git a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java new file mode 100644 index 000000000..e51d8cdf8 --- /dev/null +++ b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java @@ -0,0 +1,41 @@ +package benchmark.dynamicWorkloadGenerator; + +import benchmark.datagenerator.DataGenerator; +import benchmark.datagenerator.DataGeneratorConfig; + + +/** + * Generate dynamic workload + * 1. workload configuration generation{@link DynamicDataGeneratorConfig} + * 2. generate different workloads based on type, including + * 3. control the rate of the workload shift by setting the number of events per workload configuration + * Created by curry on 16/3/22. + */ +public abstract class DynamicWorkloadGenerator extends DataGenerator { + protected DynamicDataGeneratorConfig dynamicDataConfig; + public DynamicWorkloadGenerator(DynamicDataGeneratorConfig dynamicDataConfig) { + super(dynamicDataConfig); + this.dynamicDataConfig=dynamicDataConfig; + } + + private String[] tranToDecisionConf() { + return null; + } + + @Override + public void generateStream() { + for (int tupleNumber = 0; tupleNumber < nTuples + dynamicDataConfig.getTotalThreads(); tupleNumber++) { + if (tupleNumber%dynamicDataConfig.getCheckpoint_interval()* dynamicDataConfig.getShiftRate()* dynamicDataConfig.getTotalThreads() == 0) { + dynamicDataConfig.nextDataGeneratorConfig(); + switchConfiguration(); + } + generateTuple(); + } + } + public abstract void switchConfiguration(); + + @Override + public DynamicDataGeneratorConfig getDataConfig() { + return dynamicDataConfig; + } +} diff --git a/application/src/main/java/common/topology/transactional/initializer/GSInitializer.java b/application/src/main/java/common/topology/transactional/initializer/GSInitializer.java index 4cacd940b..880fbd6cf 100644 --- a/application/src/main/java/common/topology/transactional/initializer/GSInitializer.java +++ b/application/src/main/java/common/topology/transactional/initializer/GSInitializer.java @@ -5,6 +5,9 @@ import benchmark.datagenerator.DataGeneratorConfig; import benchmark.datagenerator.apps.GS.TPGTxnGenerator.GSTPGDataGenerator; import benchmark.datagenerator.apps.GS.TPGTxnGenerator.GSTPGDataGeneratorConfig; +import benchmark.datagenerator.apps.GS.TPGTxnGenerator.GSTPGDynamicDataGenerator; +import benchmark.datagenerator.apps.SL.TPGTxnGenerator.SLTPGDataGeneratorConfig; +import benchmark.dynamicWorkloadGenerator.DynamicDataGeneratorConfig; import common.collections.Configuration; import common.collections.OsUtils; import common.param.TxnEvent; @@ -68,14 +71,20 @@ public GSInitializer(Database db, int numberOfStates, double theta, int tthread, } protected void createTPGGenerator(Configuration config) { + if(config.getBoolean("isDynamic")) { + //TODO:add the dynamic workload dataGenerator + DynamicDataGeneratorConfig dynamicDataGeneratorConfig=new DynamicDataGeneratorConfig(); + dynamicDataGeneratorConfig.initialize(config); + configurePath(dynamicDataGeneratorConfig); + dataGenerator=new GSTPGDynamicDataGenerator(dynamicDataGeneratorConfig); + }else { + GSTPGDataGeneratorConfig dataConfig = new GSTPGDataGeneratorConfig(); + dataConfig.initialize(config); - GSTPGDataGeneratorConfig dataConfig = new GSTPGDataGeneratorConfig(); - dataConfig.initialize(config); - - configurePath(dataConfig); - dataGenerator = new GSTPGDataGenerator(dataConfig); + configurePath(dataConfig); + dataGenerator = new GSTPGDataGenerator(dataConfig); + } } - /** * Control the input file path. * TODO: think carefully which configuration shall vary. @@ -88,24 +97,33 @@ protected void createTPGGenerator(Configuration config) { * * @param dataConfig */ - private void configurePath(GSTPGDataGeneratorConfig dataConfig) { + private void configurePath(DataGeneratorConfig dataConfig) { MessageDigest digest; String subFolder = null; try { digest = MessageDigest.getInstance("SHA-256"); byte[] bytes; + if (dataConfig instanceof GSTPGDataGeneratorConfig) bytes = digest.digest(String.format("%d_%d_%d_%d_%d_%d_%d_%d_%s", dataConfig.getTotalThreads(), dataConfig.getTotalEvents(), dataConfig.getnKeyStates(), - dataConfig.NUM_ACCESS, - dataConfig.State_Access_Skewness, - dataConfig.Ratio_of_Overlapped_Keys, - dataConfig.Ratio_of_Transaction_Aborts, - dataConfig.Transaction_Length, + ((GSTPGDataGeneratorConfig) dataConfig).NUM_ACCESS, + ((GSTPGDataGeneratorConfig) dataConfig).State_Access_Skewness, + ((GSTPGDataGeneratorConfig) dataConfig).Ratio_of_Overlapped_Keys, + ((GSTPGDataGeneratorConfig) dataConfig).Ratio_of_Transaction_Aborts, + ((GSTPGDataGeneratorConfig) dataConfig).Transaction_Length, AppConfig.isCyclic) .getBytes(StandardCharsets.UTF_8)); - + else + bytes = digest.digest(String.format("%d_%d_%d_%s_%s_%s", + dataConfig.getTotalThreads(), + dataConfig.getTotalEvents(), + dataConfig.getnKeyStates(), + ((DynamicDataGeneratorConfig) dataConfig).getType(), + ((DynamicDataGeneratorConfig) dataConfig).getApp(), + AppConfig.isCyclic) + .getBytes(StandardCharsets.UTF_8)); subFolder = OsUtils.osWrapperPostFix( DatatypeConverter.printHexBinary(bytes)); } catch (Exception e) { diff --git a/application/src/main/java/common/topology/transactional/initializer/SLInitializer.java b/application/src/main/java/common/topology/transactional/initializer/SLInitializer.java index c77bb22f4..992d1aa54 100644 --- a/application/src/main/java/common/topology/transactional/initializer/SLInitializer.java +++ b/application/src/main/java/common/topology/transactional/initializer/SLInitializer.java @@ -7,6 +7,8 @@ import benchmark.datagenerator.apps.SL.OCTxnGenerator.LayeredOCDataGeneratorConfig; import benchmark.datagenerator.apps.SL.TPGTxnGenerator.SLTPGDataGenerator; import benchmark.datagenerator.apps.SL.TPGTxnGenerator.SLTPGDataGeneratorConfig; +import benchmark.datagenerator.apps.SL.TPGTxnGenerator.SLTPGDynamicDataGenerator; +import benchmark.dynamicWorkloadGenerator.DynamicDataGeneratorConfig; import common.collections.Configuration; import common.collections.OsUtils; import common.param.TxnEvent; @@ -68,12 +70,17 @@ public SLInitializer(Database db, int numberOfStates, double theta, int tthread, } protected void createTPGGenerator(Configuration config) { - - SLTPGDataGeneratorConfig dataConfig = new SLTPGDataGeneratorConfig(); - dataConfig.initialize(config); - - configurePath(dataConfig); - dataGenerator = new SLTPGDataGenerator(dataConfig); + if (config.getBoolean("isDynamic")) { + DynamicDataGeneratorConfig dynamicDataGeneratorConfig=new DynamicDataGeneratorConfig(); + dynamicDataGeneratorConfig.initialize(config); + configurePath(dynamicDataGeneratorConfig); + dataGenerator=new SLTPGDynamicDataGenerator(dynamicDataGeneratorConfig); + } else { + SLTPGDataGeneratorConfig dataConfig = new SLTPGDataGeneratorConfig(); + dataConfig.initialize(config); + configurePath(dataConfig); + dataGenerator = new SLTPGDataGenerator(dataConfig); + } } protected void createLayeredOCGenerator(Configuration config) { @@ -108,6 +115,15 @@ private void configurePath(DataGeneratorConfig dataConfig) { ((SLTPGDataGeneratorConfig) dataConfig).Ratio_of_Transaction_Aborts, AppConfig.isCyclic) .getBytes(StandardCharsets.UTF_8)); + else if(dataConfig instanceof DynamicDataGeneratorConfig) + bytes = digest.digest(String.format("%d_%d_%d_%s_%s_%s", + dataConfig.getTotalThreads(), + dataConfig.getTotalEvents(), + dataConfig.getnKeyStates(), + ((DynamicDataGeneratorConfig) dataConfig).getType(), + ((DynamicDataGeneratorConfig) dataConfig).getApp(), + AppConfig.isCyclic) + .getBytes(StandardCharsets.UTF_8)); else bytes = digest.digest(String.format("%d_%d_%d", dataConfig.getTotalThreads(), diff --git a/common/src/main/java/common/Runner.java b/common/src/main/java/common/Runner.java index 2f406c450..d2c47f2fe 100644 --- a/common/src/main/java/common/Runner.java +++ b/common/src/main/java/common/Runner.java @@ -19,8 +19,8 @@ public abstract class Runner implements IRunner { * Workload Specific Parameters. */ @Parameter(names = {"-a", "--app"}, description = "The application to be executed") -// public String application = "StreamLedger"; - public String application = "GrepSum"; + public String application = "StreamLedger"; + //public String application = "GrepSum"; @Parameter(names = {"-t", "--topology-name"}, required = false, description = "The name of the application") public String topologyName; @Parameter(names = {"--COMPUTE_COMPLEXITY"}, description = "COMPUTE_COMPLEXITY per event") @@ -99,6 +99,15 @@ public abstract class Runner implements IRunner { @Parameter(names = {"--WorkloadConfig"}, description = "WorkloadConfigs(TD,LD,PD,VDD,R_of_A,isCD,isCC,markId)") public String WorkloadConfig = "1,3,4,0.5,0.6,1,1,39999;1,6,4,0.5,0.7,1,1,79999;3,6,3,0.5,0.7,1,1,100000"; + /** + * Dynamic workload + */ + @Parameter(names = {"--workloadType"}, description = "which type of dynamic workload") + + public String workloadType = ""; + @Parameter(names = {"-shiftRate-"}, description = "control the rate of the workload shift") + public int shiftRate = 5; + /** * Benchmarking Specific Parameters. */ @@ -222,7 +231,7 @@ public void initializeCfg(HashMap config) { config.put("size_tuple", size_tuple); config.put("verbose", verbose); - /* Dynamic Workload Configuration*/ + /* Dynamic switch scheduler*/ config.put("isDynamic",isDynamic); config.put("schedulersPool",schedulerPools); config.put("defaultScheduler",scheduler); @@ -230,6 +239,11 @@ public void initializeCfg(HashMap config) { config.put("bottomLine",bottomLine); config.put("WorkloadConfig",WorkloadConfig); + /* Dynamic Workload Configuration*/ + config.put("application",application); + config.put("workloadType",workloadType); + config.put("shiftRate",shiftRate); + System.setProperty("my.log", metric_path); } } diff --git a/state-engine/src/main/java/scheduler/collector/Collector.java b/state-engine/src/main/java/scheduler/collector/Collector.java index afaf3df63..6c9a400c7 100644 --- a/state-engine/src/main/java/scheduler/collector/Collector.java +++ b/state-engine/src/main/java/scheduler/collector/Collector.java @@ -34,7 +34,7 @@ public class Collector { */ private double R_of_A; private double B_R_of_A; - private boolean isRuntime=true; + private boolean isRuntime = true; private int threadCount; private final List workloadConfig=new ArrayList<>(); private final HashMap currentWorkload=new HashMap<>(); diff --git a/stream-engine/src/main/java/execution/ExecutionManager.java b/stream-engine/src/main/java/execution/ExecutionManager.java index 9a5efa48f..20ebeceab 100644 --- a/stream-engine/src/main/java/execution/ExecutionManager.java +++ b/stream-engine/src/main/java/execution/ExecutionManager.java @@ -9,7 +9,6 @@ import execution.runtime.executorThread; import execution.runtime.spoutThread; import optimization.OptimizationManager; -import org.apache.zookeeper.txn.Txn; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import transaction.TxnManager; From ba60f1a0bf82d0d10165a103b1e37412642d9702 Mon Sep 17 00:00:00 2001 From: curryzjj Date: Fri, 18 Mar 2022 16:36:28 +0800 Subject: [PATCH 11/15] Add Application(TP and OB) --- .../apps/OB/OBTPGDynamicDataGenerator.java | 264 ++++++++++++++++++ .../apps/OB/Transaction/BuyEvent.java | 27 ++ .../apps/OB/Transaction/OBEvent.java | 32 +++ .../apps/TP/TPTPGDynamicDataGenerator.java | 166 +++++++++++ .../TP/Transaction/TollProcessingEvent.java | 27 ++ .../DynamicDataGeneratorConfig.java | 12 +- .../DynamicWorkloadGenerator.java | 7 +- application/src/main/java/combo/OBCombo.java | 76 +---- application/src/main/java/combo/TPCombo.java | 75 ++--- .../bolts/transactional/gs/GSBolt_ts.java | 6 +- .../bolts/transactional/ob/OBBolt_ts_s.java | 3 +- .../bolts/transactional/tp/TPBolt_ts_s.java | 3 +- .../java/common/datatype/PositionReport.java | 6 +- .../datatype/internal/AccidentTuple.java | 4 +- .../datatype/internal/AvgSpeedTuple.java | 4 +- .../internal/AvgVehicleSpeedTuple.java | 4 +- .../common/datatype/internal/CountTuple.java | 4 +- .../common/datatype/internal/LavTuple.java | 4 +- .../datatype/util/ISegmentIdentifier.java | 2 +- .../datatype/util/SegmentIdentifier.java | 8 +- .../main/java/common/param/lr/LREvent.java | 9 +- .../main/java/common/param/ob/AlertEvent.java | 10 +- .../java/common/param/ob/BuyingEvent.java | 40 ++- .../java/common/param/ob/ToppingEvent.java | 9 +- .../main/java/common/sink/MeasureSink.java | 22 ++ .../topology/transactional/OnlineBiding.java | 5 +- .../transactional/TollProcessing.java | 5 +- .../initializer/OBInitializer.java | 222 +++++++++------ .../initializer/TPInitializer.java | 152 +++++++++- .../main/java/runners/MorphStreamRunner.java | 29 +- common/src/main/java/common/Runner.java | 19 +- .../config/TollProcessing.properties | 2 +- pom.xml | 43 --- .../src/main/java/common/param/TxnEvent.java | 6 + .../src/main/java/scheduler/Request.java | 11 + .../java/scheduler/impl/og/OGScheduler.java | 68 ++++- .../java/scheduler/impl/op/OPScheduler.java | 59 +++- .../struct/og/TaskPrecedenceGraph.java | 19 +- .../struct/op/TaskPrecedenceGraph.java | 19 +- .../impl/TxnManagerDedicatedAsy.java | 2 +- 40 files changed, 1170 insertions(+), 315 deletions(-) create mode 100644 application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java create mode 100644 application/src/main/java/benchmark/datagenerator/apps/OB/Transaction/BuyEvent.java create mode 100644 application/src/main/java/benchmark/datagenerator/apps/OB/Transaction/OBEvent.java create mode 100644 application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java create mode 100644 application/src/main/java/benchmark/datagenerator/apps/TP/Transaction/TollProcessingEvent.java diff --git a/application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java new file mode 100644 index 000000000..e510cfc7e --- /dev/null +++ b/application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java @@ -0,0 +1,264 @@ +package benchmark.datagenerator.apps.OB; + +import benchmark.datagenerator.Event; +import benchmark.datagenerator.apps.OB.Transaction.BuyEvent; +import benchmark.datagenerator.apps.OB.Transaction.OBEvent; +import benchmark.dynamicWorkloadGenerator.DynamicDataGeneratorConfig; +import benchmark.dynamicWorkloadGenerator.DynamicWorkloadGenerator; +import common.tools.FastZipfGenerator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import utils.AppConfig; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Random; + +import static common.CONTROL.enable_log; +import static common.CONTROL.enable_states_partition; + +/** + * Dynamic data generator for OnlineBidding + * Created by curry on 17/3/22. + */ +public class OBTPGDynamicDataGenerator extends DynamicWorkloadGenerator { + private static final Logger LOG= LoggerFactory.getLogger(OBTPGDynamicDataGenerator.class); + private int NUM_ACCESS; // transaction length, 4 or 8 or longer + private int State_Access_Skewness; // ratio of state access, following zipf distribution + private int Ratio_of_Transaction_Aborts; // ratio of transaction aborts, fail the transaction or not. i.e. transfer amount might be invalid. + private int Ratio_of_Overlapped_Keys; // ratio of overlapped keys in transactions, which affects the dependencies and circulars. + private int Transaction_Length; + private int tthread; + private int Ratio_of_Buying; + // control the number of txns overlap with each other. + private ArrayList generatedKeys = new ArrayList<>(); + // independent transactions. + private boolean isUnique = false; + private FastZipfGenerator keyZipf; + + private int floor_interval; + public FastZipfGenerator[] partitionedKeyZipf; + private Random random = new Random(0); // the transaction type decider + public transient FastZipfGenerator p_generator; // partition generator + private HashMap nGeneratedIds = new HashMap<>(); + private ArrayList events; + private int eventID = 0; + private HashMap idToLevel = new HashMap<>(); + public OBTPGDynamicDataGenerator(DynamicDataGeneratorConfig dynamicDataConfig) { + super(dynamicDataConfig); + events = new ArrayList<>(nTuples); + } + @Override + protected void generateTuple() { + Event event; + int next = random.nextInt(100); + if(next < Ratio_of_Buying) { + event = randomBuyingEvent(); + } else { + next = random.nextInt(100); + if (next > 50){ + event = randomOBEvent(1); + } else { + event = randomOBEvent(0); + } + } + events.add(event); + } + + @Override + public void switchConfiguration() { + State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; + NUM_ACCESS = dynamicDataConfig.NUM_ACCESS; + Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; + Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; + Transaction_Length = dynamicDataConfig.Transaction_Length; + Ratio_of_Buying=dynamicDataConfig.Ratio_Of_Buying; + + int nKeyState = dynamicDataConfig.getnKeyStates(); + int MAX_LEVEL = 256; + for (int i = 0; i < nKeyState; i++) { + idToLevel.put(i, i% MAX_LEVEL); + } + keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); + configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); + p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); + } + + @Override + public void dumpGeneratedDataToFile() { + if (enable_log) LOG.info("++++++" + nGeneratedIds.size()); + + if (enable_log) LOG.info("Dumping transactions..."); + try { + dataOutputHandler.sinkEvents(events); + } catch (IOException e) { + e.printStackTrace(); + } + + File versionFile = new File(dynamicDataConfig.getRootPath().substring(0, dynamicDataConfig.getRootPath().length() - 1) + + String.format("_%d.txt", dynamicDataConfig.getTotalEvents())); + try { + versionFile.createNewFile(); + FileWriter fileWriter = new FileWriter(versionFile); + fileWriter.write(String.format("Total number of threads : %d\n", dynamicDataConfig.getTotalThreads())); + fileWriter.write(String.format("Total Events : %d\n", dynamicDataConfig.getTotalEvents())); + fileWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void configure_store(double scale_factor, double theta, int tthread, int numItems) { + floor_interval = (int) Math.floor(numItems / (double) tthread);//NUM_ITEMS / tthread; + partitionedKeyZipf = new FastZipfGenerator[tthread];//overhead_total number of working threads. + for (int i = 0; i < tthread; i++) { + partitionedKeyZipf[i] = new FastZipfGenerator((int) (floor_interval * scale_factor), theta, i * floor_interval, 12345678); + } + } + + private Event randomOBEvent(int isAlert) { + int[] keys = new int[NUM_ACCESS*Transaction_Length]; + int writeLevel = -1; + if (!isUnique) { + if (enable_states_partition) { + for (int j = 0; j < Transaction_Length; j++) { + int partitionId = key_to_partition(p_generator.next()); + for (int i = 0; i < NUM_ACCESS; i++) { + int offset = j * NUM_ACCESS + i; + if (AppConfig.isCyclic) { + int key = getKey(partitionedKeyZipf[partitionId], partitionId, generatedKeys); + if (offset % NUM_ACCESS == 0) { + // make sure this one is different with other write key + for (int k = 0; k < j; k++) { + while (keys[k*NUM_ACCESS] == key) { + key = getKey(partitionedKeyZipf[partitionId], partitionId, generatedKeys); + } + } + } + keys[offset] = key; + } else { + // TODO: correct it later + keys[offset] = getKey(partitionedKeyZipf[partitionId], partitionId, generatedKeys); + if (i == 0) { + while (idToLevel.get(keys[offset]) == 0) { + keys[offset] = getKey(partitionedKeyZipf[partitionId], partitionId, generatedKeys); + } + writeLevel = idToLevel.get(keys[offset]); + } else { + while (writeLevel <= idToLevel.get(keys[offset])) { + keys[offset] = getKey(partitionedKeyZipf[partitionId], partitionId, generatedKeys); + } + } + } + partitionId = (partitionId + 1) % dynamicDataConfig.getTotalThreads(); + } + } + } else { + for (int i = 0; i < NUM_ACCESS; i++) { + if (AppConfig.isCyclic) { + keys[i] = getKey(keyZipf, generatedKeys); + } else { + keys[i] = getKey(keyZipf, generatedKeys); + if (i == 0) { + while (idToLevel.get(keys[i]) == 0) { + keys[i] = getKey(keyZipf, generatedKeys); + } + writeLevel = idToLevel.get(keys[i]); + } + while (writeLevel <= idToLevel.get(keys[i])) { + keys[i] = getKey(keyZipf, generatedKeys); + } + } + } + } + } else { + // TODO: add transaction length logic + for (int i = 0; i generatedKeys) { + int key; + key = zipfGenerator.next(); + int next = random.nextInt(100); + if (next < Ratio_of_Overlapped_Keys) { // randomly select a key from existing keyset. + if (!generatedKeys.isEmpty()) { + int counter = 0; + key = generatedKeys.get(random.nextInt(generatedKeys.size())); + while (key / floor_interval != partitionId) { + key = generatedKeys.get(random.nextInt(generatedKeys.size())); + counter++; + if (counter >= partitionId) { + key = zipfGenerator.next(); + break; + } + } + } + } + return key; + } + + private int getKey(FastZipfGenerator zipfGenerator, ArrayList generatedKeys) { + int srcKey; + srcKey = zipfGenerator.next(); + int next = random.nextInt(100); + if (next < Ratio_of_Overlapped_Keys) { // randomly select a key from existing keyset. + if (!generatedKeys.isEmpty()) { + srcKey = generatedKeys.get(zipfGenerator.next() % generatedKeys.size()); + } + } + return srcKey; + } + + private int getUniqueKey(FastZipfGenerator zipfGenerator, ArrayList generatedKeys) { + int key; + key = zipfGenerator.next(); + while (generatedKeys.contains(key)) { + key = zipfGenerator.next(); + } + generatedKeys.add(key); + return key; + } +} diff --git a/application/src/main/java/benchmark/datagenerator/apps/OB/Transaction/BuyEvent.java b/application/src/main/java/benchmark/datagenerator/apps/OB/Transaction/BuyEvent.java new file mode 100644 index 000000000..af3d4d7c4 --- /dev/null +++ b/application/src/main/java/benchmark/datagenerator/apps/OB/Transaction/BuyEvent.java @@ -0,0 +1,27 @@ +package benchmark.datagenerator.apps.OB.Transaction; + +import benchmark.datagenerator.Event; + +/** + * OnlineBidding Event for (buy) + * Created by curry on 17/3/22. + */ +public class BuyEvent extends Event { + private final int id; + private final int key; + private final boolean isAbort; + public BuyEvent(int id, int key, boolean isAbort) { + this.id = id; + this.key = key; + this.isAbort = isAbort; + } + + @Override + public String toString() { + StringBuilder str = new StringBuilder(String.valueOf(id)); + str.append(",").append(key); + str.append(",").append(isAbort); + + return str.toString(); + } +} diff --git a/application/src/main/java/benchmark/datagenerator/apps/OB/Transaction/OBEvent.java b/application/src/main/java/benchmark/datagenerator/apps/OB/Transaction/OBEvent.java new file mode 100644 index 000000000..b8f437a5e --- /dev/null +++ b/application/src/main/java/benchmark/datagenerator/apps/OB/Transaction/OBEvent.java @@ -0,0 +1,32 @@ +package benchmark.datagenerator.apps.OB.Transaction; + +import benchmark.datagenerator.Event; + +/** + * OnlineBidding Event for (alert and top) + * Created by curry on 17/3/22. + */ +public class OBEvent extends Event { + private final int id; + private final int[] keys; + private final boolean isAbort; + private final int isAlert; + + public OBEvent(int id, int[] keys, boolean isAbort, int isAlert){ + this.id = id; + this.keys = keys; + this.isAbort = isAbort; + this.isAlert=isAlert; + } + @Override + public String toString() { + StringBuilder str = new StringBuilder(String.valueOf(id)); + for (int key : keys) { + str.append(",").append(key); + } + str.append(",").append(isAbort); + str.append(",").append(isAlert); + + return str.toString(); + } +} diff --git a/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java new file mode 100644 index 000000000..9eff29c23 --- /dev/null +++ b/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java @@ -0,0 +1,166 @@ +package benchmark.datagenerator.apps.TP; + +import benchmark.datagenerator.Event; +import benchmark.datagenerator.apps.TP.Transaction.TollProcessingEvent; +import benchmark.dynamicWorkloadGenerator.DynamicDataGeneratorConfig; +import benchmark.dynamicWorkloadGenerator.DynamicWorkloadGenerator; +import common.tools.FastZipfGenerator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Random; + +import static common.CONTROL.enable_log; +import static common.CONTROL.enable_states_partition; + +public class TPTPGDynamicDataGenerator extends DynamicWorkloadGenerator { + private static final Logger LOG = LoggerFactory.getLogger(TPTPGDynamicDataGenerator.class); + private int State_Access_Skewness; // ratio of state access, following zipf distribution + private int Ratio_of_Transaction_Aborts; // ratio of transaction aborts, fail the transaction or not. i.e. transfer amount might be invalid. + private int Ratio_of_Overlapped_Keys; // ratio of overlapped keys in transactions, which affects the dependencies and circulars. + // control the number of txns overlap with each other. + private ArrayList generatedKeys = new ArrayList<>(); + // independent transactions. + private boolean isUnique = false; + private FastZipfGenerator keyZipf; + + private int floor_interval; + public FastZipfGenerator[] partitionedKeyZipf; + private Random random = new Random(0); // the transaction type decider + public transient FastZipfGenerator p_generator; // partition generator + private HashMap nGeneratedIds = new HashMap<>(); + private ArrayList events; + private int eventID = 0; + private HashMap idToLevel = new HashMap<>(); + public TPTPGDynamicDataGenerator(DynamicDataGeneratorConfig dynamicDataConfig) { + super(dynamicDataConfig); + events = new ArrayList<>(nTuples); + } + + @Override + public void switchConfiguration() { + State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; + Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; + Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; + int nKeyState = dynamicDataConfig.getnKeyStates(); + int MAX_LEVEL = 256; + for (int i = 0; i < nKeyState; i++) { + idToLevel.put(i, i% MAX_LEVEL); + } + keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); + configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); + p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); + } + public void configure_store(double scale_factor, double theta, int tthread, int numItems) { + floor_interval = (int) Math.floor(numItems / (double) tthread);//NUM_ITEMS / tthread; + partitionedKeyZipf = new FastZipfGenerator[tthread];//overhead_total number of working threads. + for (int i = 0; i < tthread; i++) { + partitionedKeyZipf[i] = new FastZipfGenerator((int) (floor_interval * scale_factor), theta, i * floor_interval, 12345678); + } + } + + @Override + protected void generateTuple() { + Event event; + event = randomTPEvent(); + this.events.add(event); + } + + @Override + public void dumpGeneratedDataToFile() { + if (enable_log) LOG.info("++++++" + nGeneratedIds.size()); + + if (enable_log) LOG.info("Dumping transactions..."); + try { + dataOutputHandler.sinkEvents(events); + } catch (IOException e) { + e.printStackTrace(); + } + + File versionFile = new File(dynamicDataConfig.getRootPath().substring(0, dynamicDataConfig.getRootPath().length() - 1) + + String.format("_%d.txt", dynamicDataConfig.getTotalEvents())); + try { + versionFile.createNewFile(); + FileWriter fileWriter = new FileWriter(versionFile); + fileWriter.write(String.format("Total number of threads : %d\n", dynamicDataConfig.getTotalThreads())); + fileWriter.write(String.format("Total Events : %d\n", dynamicDataConfig.getTotalEvents())); + fileWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + private Event randomTPEvent() { + int id; + if (!isUnique) { + if (enable_states_partition) { + int partitionId = key_to_partition(p_generator.next()); + id = getKey(partitionedKeyZipf[partitionId],partitionId,generatedKeys); + } else { + id = getKey(keyZipf,generatedKeys); + } + } else { + id = getUniqueKey(keyZipf,generatedKeys); + } + nGeneratedIds.put(id, nGeneratedIds.getOrDefault(id, 0) + 1); + Event t; + //TODO: the "isAbort" does not matter, since the operation of Toll processing never abort + if (random.nextInt(10000) < Ratio_of_Transaction_Aborts){ + t = new TollProcessingEvent(eventID,id,true); + }else { + t = new TollProcessingEvent(eventID,id,false); + } + eventID++; + return t; + } + public int key_to_partition(int key) { + return (int) Math.floor((double) key / floor_interval); + } + + private int getKey(FastZipfGenerator zipfGenerator, int partitionId, ArrayList generatedKeys) { + int key; + key = zipfGenerator.next(); + int next = random.nextInt(100); + if (next < Ratio_of_Overlapped_Keys) { // randomly select a key from existing keyset. + if (!generatedKeys.isEmpty()) { + int counter = 0; + key = generatedKeys.get(random.nextInt(generatedKeys.size())); + while (key / floor_interval != partitionId) { + key = generatedKeys.get(random.nextInt(generatedKeys.size())); + counter++; + if (counter >= partitionId) { + key = zipfGenerator.next(); + break; + } + } + } + } + return key; + } + + private int getKey(FastZipfGenerator zipfGenerator, ArrayList generatedKeys) { + int srcKey; + srcKey = zipfGenerator.next(); + int next = random.nextInt(100); + if (next < Ratio_of_Overlapped_Keys) { // randomly select a key from existing keyset. + if (!generatedKeys.isEmpty()) { + srcKey = generatedKeys.get(zipfGenerator.next() % generatedKeys.size()); + } + } + return srcKey; + } + + private int getUniqueKey(FastZipfGenerator zipfGenerator, ArrayList generatedKeys) { + int key; + key = zipfGenerator.next(); + while (generatedKeys.contains(key)) { + key = zipfGenerator.next(); + } + generatedKeys.add(key); + return key; + } +} diff --git a/application/src/main/java/benchmark/datagenerator/apps/TP/Transaction/TollProcessingEvent.java b/application/src/main/java/benchmark/datagenerator/apps/TP/Transaction/TollProcessingEvent.java new file mode 100644 index 000000000..d58cf3914 --- /dev/null +++ b/application/src/main/java/benchmark/datagenerator/apps/TP/Transaction/TollProcessingEvent.java @@ -0,0 +1,27 @@ +package benchmark.datagenerator.apps.TP.Transaction; + +import benchmark.datagenerator.Event; +import common.datatype.PositionReport; + +/** + * Toll Processing data + * Created by curry on 17/3/22. + */ +public class TollProcessingEvent extends Event { + private final int id; + private final int segmentId; + private final boolean isAbort; + public TollProcessingEvent(int id, int segmentId, boolean isAbort) { + this.id = id; + this.segmentId = segmentId; + this.isAbort = isAbort; + } + @Override + public String toString() { + StringBuilder str = new StringBuilder(String.valueOf(id)); + str.append(",").append(segmentId); + str.append(",").append(isAbort); + + return str.toString(); + } +} diff --git a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java index 2f9680259..6f0d955ad 100644 --- a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java +++ b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java @@ -1,13 +1,8 @@ package benchmark.dynamicWorkloadGenerator; import benchmark.datagenerator.DataGeneratorConfig; -import benchmark.datagenerator.apps.GS.TPGTxnGenerator.GSTPGDataGeneratorConfig; -import benchmark.datagenerator.apps.SL.TPGTxnGenerator.SLTPGDataGeneratorConfig; import common.collections.Configuration; -import java.util.ArrayList; -import java.util.List; - /** * Generate dynamic workload configuration * 1. workload configuration generation @@ -27,6 +22,7 @@ public class DynamicDataGeneratorConfig extends DataGeneratorConfig { public int Ratio_of_Transaction_Aborts; public int Transaction_Length; public int Ratio_Of_Deposit; + public int Ratio_Of_Buying; public void initialize(Configuration config) { super.initialize(config); @@ -40,6 +36,7 @@ public void initialize(Configuration config) { Ratio_of_Transaction_Aborts = config.getInt("Ratio_of_Transaction_Aborts", 0); Transaction_Length = config.getInt("Transaction_Length", 1); Ratio_Of_Deposit = config.getInt("Ratio_Of_Deposit", 0); + Ratio_Of_Buying=config.getInt("Ratio_Of_Buying",0); } public int getCheckpoint_interval() { @@ -53,8 +50,9 @@ public int getShiftRate() { * Generate the configuration based on type and application * @return */ - public void nextDataGeneratorConfig() { - + public boolean nextDataGeneratorConfig() { + //TODO: how to switch + return false; } public String getType() { diff --git a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java index e51d8cdf8..eb68bf7d8 100644 --- a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java +++ b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java @@ -24,10 +24,13 @@ private String[] tranToDecisionConf() { @Override public void generateStream() { + //Init the Configuration + switchConfiguration(); for (int tupleNumber = 0; tupleNumber < nTuples + dynamicDataConfig.getTotalThreads(); tupleNumber++) { if (tupleNumber%dynamicDataConfig.getCheckpoint_interval()* dynamicDataConfig.getShiftRate()* dynamicDataConfig.getTotalThreads() == 0) { - dynamicDataConfig.nextDataGeneratorConfig(); - switchConfiguration(); + if (dynamicDataConfig.nextDataGeneratorConfig()) { + switchConfiguration(); + } } generateTuple(); } diff --git a/application/src/main/java/combo/OBCombo.java b/application/src/main/java/combo/OBCombo.java index 11c7404af..17e8f41bb 100644 --- a/application/src/main/java/combo/OBCombo.java +++ b/application/src/main/java/combo/OBCombo.java @@ -1,5 +1,6 @@ package combo; +import benchmark.DataHolder; import common.bolts.transactional.ob.*; import common.collections.Configuration; import common.collections.OsUtils; @@ -103,65 +104,17 @@ protected void show_stats() { @Override public void loadEvent(String file_name, Configuration config, TopologyContext context, OutputCollector collector) { - String event_path = Event_Path - + OsUtils.OS_wrapper("enable_states_partition=" + enable_states_partition); - if (Files.notExists(Paths.get(event_path + OsUtils.OS_wrapper(file_name)))) - System.exit(-1); - Scanner sc; - try { - sc = new Scanner(new File(event_path + OsUtils.OS_wrapper(file_name))); - int i = 0; - Object event; - for (int j = 0; j < taskId; j++) { - sc.nextLine(); - } - while (sc.hasNextLine()) { - String read = sc.nextLine(); - String[] split = read.split(split_exp); - if (split[4].endsWith("BuyingEvent")) {//BuyingEvent - event = new BuyingEvent( - Integer.parseInt(split[0]), //bid - split[2], //bid_array - Integer.parseInt(split[1]),//pid - Integer.parseInt(split[3]),//num_of_partition - split[5],//key_array - split[6],//price_array - split[7] //qty_array - ); - } else if (split[4].endsWith("AlertEvent")) {//AlertEvent - event = new AlertEvent( - Integer.parseInt(split[0]), //bid - split[2], // bid_array - Integer.parseInt(split[1]),//pid - Integer.parseInt(split[3]),//num_of_partition - Integer.parseInt(split[5]), //num_access - split[6],//key_array - split[7]//price_array - ); - } else { - event = new ToppingEvent( - Integer.parseInt(split[0]), //bid - split[2], Integer.parseInt(split[1]), //pid - //bid_array - Integer.parseInt(split[3]),//num_of_partition - Integer.parseInt(split[5]), //num_access - split[6],//key_array - split[7] //top_array - ); - } -// db.eventManager.put(input_event, Integer.parseInt(split[0])); - myevents[i++] = event; - if (i == num_events_per_thread) break; - for (int j = 0; j < (tthread - 1) * combo_bid_size; j++) { - if (sc.hasNextLine()) - sc.nextLine();//skip un-related. - } - } - } catch (FileNotFoundException e) { - e.printStackTrace(); + int storageIndex = 0; + //Load OnlineBiding Events. + for (int index = taskId; index < DataHolder.events.size(); ) { + TxnEvent event = DataHolder.events.get(index).cloneEvent(); + mybids[storageIndex] = event.getBid(); + myevents[storageIndex++] = event; + if (storageIndex == num_events_per_thread) + break; + index += tthread * combo_bid_size; } - if (enable_log) - show_stats(); + assert (storageIndex == num_events_per_thread); } @Override @@ -186,7 +139,7 @@ public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { break; } case CCOption_TStream: {//T-Stream - bolt = new OBBolt_ts(0, sink); + bolt = new OBBolt_ts_s(0, sink); break; } case CCOption_SStore: {//SStore @@ -197,9 +150,8 @@ public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { } //do preparation. bolt.prepare(config, context, collector); - if (enable_shared_state) - bolt.loadDB(config, context, collector); - loadEvent("OB_Events" + tthread, config, context, collector); + bolt.loadDB(config, context, collector); + loadEvent(config.getString("rootFilePath"), config, context, collector); // bolt.sink.batch_number_per_wm = batch_number_per_wm; } } \ No newline at end of file diff --git a/application/src/main/java/combo/TPCombo.java b/application/src/main/java/combo/TPCombo.java index 1432e88ae..f4e4dce57 100644 --- a/application/src/main/java/combo/TPCombo.java +++ b/application/src/main/java/combo/TPCombo.java @@ -1,11 +1,11 @@ package combo; +import benchmark.DataHolder; import common.bolts.transactional.tp.*; import common.collections.Configuration; -import common.collections.OsUtils; -import common.constants.BaseConstants; import common.datatype.AbstractLRBTuple; import common.datatype.PositionReport; +import common.param.TxnEvent; import common.param.lr.LREvent; import components.context.TopologyContext; import execution.ExecutionGraph; @@ -14,12 +14,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileNotFoundException; import java.util.ArrayDeque; import java.util.HashMap; import java.util.Map; -import java.util.Scanner; import static common.CONTROL.*; import static content.Content.*; @@ -27,7 +24,7 @@ public class TPCombo extends SPOUTCombo { private static final Logger LOG = LoggerFactory.getLogger(TPCombo.class); private static final long serialVersionUID = -2394340130331865581L; - HashMap keys = new HashMap(); + HashMap keys = new HashMap(); DescriptiveStatistics stats = new DescriptiveStatistics(); int concurrency = 0; int pre_concurrency = 0; @@ -40,8 +37,8 @@ public TPCombo() { } private boolean check_conflict(LREvent pre_event, LREvent event) { - Short pre_segment = pre_event.getPOSReport().getSegment(); - Short current_segment = event.getPOSReport().getSegment(); + Integer pre_segment = pre_event.getPOSReport().getSegment(); + Integer current_segment = event.getPOSReport().getSegment(); return pre_segment.equals(current_segment); } @@ -55,7 +52,7 @@ private boolean conflict(LREvent event) { protected void show_stats() { for (Object myevent : myevents) { - Short segment = ((LREvent) myevent).getPOSReport().getSegment(); + Integer segment = ((LREvent) myevent).getPOSReport().getSegment(); stats.addValue(segment); boolean containsKey = keys.containsKey(segment); if (containsKey) { @@ -64,7 +61,7 @@ protected void show_stats() { keys.put(segment, 1); } } - for (Map.Entry entry : keys.entrySet()) { + for (Map.Entry entry : keys.entrySet()) { LOG.info("SEGMENT:" + entry.getKey() + " " + "Counter:" + entry.getValue()); } LOG.info(stats.toString()); @@ -102,7 +99,7 @@ protected Object create_new_event(String record, int bid) { Integer.parseInt(token[4]), // xway Short.parseShort(token[5]), // lane Short.parseShort(token[6]), // direction - Short.parseShort(token[7]), // segment + Integer.parseInt(token[7]), // segment Integer.parseInt(token[8])), tthread, bid) @@ -115,33 +112,17 @@ protected Object create_new_event(String record, int bid) { @Override public void loadEvent(String file_name, Configuration config, TopologyContext context, OutputCollector collector) { - int i = 0; - int bid = 0; - Scanner sc = null; - try { - sc = new Scanner(new File(file_name)); - for (int j = 0; j < taskId; j++) { - sc.nextLine(); - } - while (sc.hasNextLine() && bid < config.getInt("totalEvents") ) { - String record = sc.nextLine(); - Object event = create_new_event(record, bid); - if (event == null) { - } else { - myevents[i++] = event; - bid++; - if (i == num_events_per_thread) break; - for (int j = 0; j < (tthread - 1) * combo_bid_size; j++) { - if (sc.hasNextLine()) - sc.nextLine();//skip un-related. - } - } - } - if (enable_debug) - show_stats(); - } catch (FileNotFoundException e) { - e.printStackTrace(); + int storageIndex = 0; + //Load OnlineBiding Events. + for (int index = taskId; index < DataHolder.events.size(); ) { + TxnEvent event = DataHolder.events.get(index).cloneEvent(); + mybids[storageIndex] = event.getBid(); + myevents[storageIndex++] = event; + if (storageIndex == num_events_per_thread) + break; + index += tthread * combo_bid_size; } + assert (storageIndex == num_events_per_thread); } @Override @@ -166,7 +147,7 @@ public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { break; } case CCOption_TStream: {//T-Stream - bolt = new TPBolt_ts(0, sink); + bolt = new TPBolt_ts_s(0, sink); break; } case CCOption_SStore: {//SStore @@ -177,22 +158,8 @@ public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { } //do preparation. bolt.prepare(config, context, collector); - if (enable_shared_state) - bolt.loadDB(config, context, collector); - String OS_prefix = null; - if (OsUtils.isWindows()) { - OS_prefix = "win."; - } else { - OS_prefix = "unix."; - } - String path; - if (OsUtils.isMac()) { - path = config.getString(getConfigKey(OS_prefix.concat(BaseConstants.BaseConf.SPOUT_TEST_PATH))); - } else { - path = config.getString(getConfigKey(OS_prefix.concat(BaseConstants.BaseConf.SPOUT_PATH))); - } - String file = System.getProperty("user.home").concat("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/data/app/").concat(path); - loadEvent(file, config, context, collector); + bolt.loadDB(config, context, collector); + loadEvent(config.getString("rootFilePath"), config, context, collector); sink.checkpoint_interval = checkpoint_interval; } } \ No newline at end of file diff --git a/application/src/main/java/common/bolts/transactional/gs/GSBolt_ts.java b/application/src/main/java/common/bolts/transactional/gs/GSBolt_ts.java index c6fab73f4..66f0a7c02 100644 --- a/application/src/main/java/common/bolts/transactional/gs/GSBolt_ts.java +++ b/application/src/main/java/common/bolts/transactional/gs/GSBolt_ts.java @@ -120,7 +120,7 @@ void READ_POST() throws InterruptedException { READ_POST(event); } } - + int i=0; @Override public void execute(Tuple in) throws InterruptedException, DatabaseException, BrokenBarrierException { @@ -132,7 +132,11 @@ public void execute(Tuple in) throws InterruptedException, DatabaseException, Br { MeasureTools.BEGIN_TXN_TIME_MEASURE(thread_Id); { + if (i==1){ + System.out.println(); + } transactionManager.start_evaluate(thread_Id, in.getBID(), num_events);//start lazy evaluation in transaction manager. + i++; READ_REQUEST_CORE(); } MeasureTools.END_TXN_TIME_MEASURE(thread_Id); diff --git a/application/src/main/java/common/bolts/transactional/ob/OBBolt_ts_s.java b/application/src/main/java/common/bolts/transactional/ob/OBBolt_ts_s.java index 044b5fec1..5b1d63b6f 100644 --- a/application/src/main/java/common/bolts/transactional/ob/OBBolt_ts_s.java +++ b/application/src/main/java/common/bolts/transactional/ob/OBBolt_ts_s.java @@ -31,7 +31,7 @@ public class OBBolt_ts_s extends OBBolt { private static final Logger LOG= LoggerFactory.getLogger(OBBolt_ts_s.class); - ArrayDeque buyingEvents; + ArrayDeque buyingEvents = new ArrayDeque<>(); private int alertEvents = 0, toppingEvents = 0; public OBBolt_ts_s(int fid, SINKCombo sink) { super(LOG, fid, sink); @@ -73,6 +73,7 @@ public void execute(Tuple in) throws InterruptedException, DatabaseException, Br alertEvents = 0; toppingEvents = 0; } + MeasureTools.END_TOTAL_TIME_MEASURE_TS(thread_Id,readSize); }else { execute_ts_normal(in); } diff --git a/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts_s.java b/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts_s.java index ef8c32f15..764a2db88 100644 --- a/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts_s.java +++ b/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts_s.java @@ -37,13 +37,12 @@ public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { super.initialize(thread_Id, thisTaskId, graph); transactionManager=new TxnManagerTStream(db.getStorageManager(), this.context.getThisComponentId(),thread_Id, NUM_SEGMENTS,this.context.getThisComponent().getNumTasks(),config.getString("scheduler","BF")); - LREvents=new ArrayDeque<>(); + LREvents = new ArrayDeque<>(); } public void loadDB(Map conf, TopologyContext context, OutputCollector collector) { loadDB(context.getThisTaskId() - context.getThisComponent().getExecutorList().get(0).getExecutorID() , context.getGraph()); } - @Override public void execute(Tuple in) throws InterruptedException, DatabaseException, BrokenBarrierException { if (in.isMarker()){ diff --git a/application/src/main/java/common/datatype/PositionReport.java b/application/src/main/java/common/datatype/PositionReport.java index e512ad2f5..5e0d7c11a 100644 --- a/application/src/main/java/common/datatype/PositionReport.java +++ b/application/src/main/java/common/datatype/PositionReport.java @@ -87,7 +87,7 @@ public PositionReport() { * @param position the horizontal position on the expressway */ public PositionReport(Short time, Integer vid, Integer speed, Integer xway, Short lane, Short direction, - Short segment, Integer position) { + Integer segment, Integer position) { super(AbstractLRBTuple.POSITION_REPORT, time, vid); assert (speed != null); assert (xway != null); @@ -165,8 +165,8 @@ public final Short getDirection() { * @return the VID of this position report */ @Override - public final Short getSegment() { - return (Short) super.get(SEG_IDX); + public final int getSegment() { + return (int) super.get(SEG_IDX); } /** diff --git a/application/src/main/java/common/datatype/internal/AccidentTuple.java b/application/src/main/java/common/datatype/internal/AccidentTuple.java index 1151e483f..5df0267e2 100644 --- a/application/src/main/java/common/datatype/internal/AccidentTuple.java +++ b/application/src/main/java/common/datatype/internal/AccidentTuple.java @@ -126,8 +126,8 @@ public final Integer getXWay() { * @return the segment of this tuple */ @Override - public final Short getSegment() { - return (Short) super.get(SEG_IDX); + public final int getSegment() { + return (Integer) super.get(SEG_IDX); } /** diff --git a/application/src/main/java/common/datatype/internal/AvgSpeedTuple.java b/application/src/main/java/common/datatype/internal/AvgSpeedTuple.java index 38164a852..35ceb14b3 100644 --- a/application/src/main/java/common/datatype/internal/AvgSpeedTuple.java +++ b/application/src/main/java/common/datatype/internal/AvgSpeedTuple.java @@ -124,8 +124,8 @@ public final Integer getXWay() { * @return the VID of this tuple */ @Override - public final Short getSegment() { - return (Short) super.get(SEG_IDX); + public final int getSegment() { + return (Integer) super.get(SEG_IDX); } /** diff --git a/application/src/main/java/common/datatype/internal/AvgVehicleSpeedTuple.java b/application/src/main/java/common/datatype/internal/AvgVehicleSpeedTuple.java index 8c6d6e06f..24d8affda 100644 --- a/application/src/main/java/common/datatype/internal/AvgVehicleSpeedTuple.java +++ b/application/src/main/java/common/datatype/internal/AvgVehicleSpeedTuple.java @@ -170,8 +170,8 @@ public final Integer getXWay() { * @return the VID of this tuple */ @Override - public final Short getSegment() { - return (Short) super.get(SEG_IDX); + public final int getSegment() { + return (Integer) super.get(SEG_IDX); } /** diff --git a/application/src/main/java/common/datatype/internal/CountTuple.java b/application/src/main/java/common/datatype/internal/CountTuple.java index cac91d9c4..910e66a75 100644 --- a/application/src/main/java/common/datatype/internal/CountTuple.java +++ b/application/src/main/java/common/datatype/internal/CountTuple.java @@ -175,8 +175,8 @@ public final Integer getXWay() { * @return the VID of this tuple */ @Override - public final Short getSegment() { - return (Short) super.get(SEG_IDX); + public final int getSegment() { + return (Integer) super.get(SEG_IDX); } /** diff --git a/application/src/main/java/common/datatype/internal/LavTuple.java b/application/src/main/java/common/datatype/internal/LavTuple.java index a9181a5a9..abd99790b 100644 --- a/application/src/main/java/common/datatype/internal/LavTuple.java +++ b/application/src/main/java/common/datatype/internal/LavTuple.java @@ -142,8 +142,8 @@ public final Integer getXWay() { * @return the segment of this tuple */ @Override - public final Short getSegment() { - return (Short) super.get(SEG_IDX); + public final int getSegment() { + return (Integer) super.get(SEG_IDX); } /** diff --git a/application/src/main/java/common/datatype/util/ISegmentIdentifier.java b/application/src/main/java/common/datatype/util/ISegmentIdentifier.java index 59ffc88e1..f2a80a4a2 100644 --- a/application/src/main/java/common/datatype/util/ISegmentIdentifier.java +++ b/application/src/main/java/common/datatype/util/ISegmentIdentifier.java @@ -28,7 +28,7 @@ public interface ISegmentIdentifier extends Serializable { Integer getXWay(); - Short getSegment(); + int getSegment(); Short getDirection(); } diff --git a/application/src/main/java/common/datatype/util/SegmentIdentifier.java b/application/src/main/java/common/datatype/util/SegmentIdentifier.java index 75afca802..9d145ed92 100644 --- a/application/src/main/java/common/datatype/util/SegmentIdentifier.java +++ b/application/src/main/java/common/datatype/util/SegmentIdentifier.java @@ -35,7 +35,7 @@ public class SegmentIdentifier implements ISegmentIdentifier { /** * Seg (0...99) identifies the mile-long segment from which the position report is emitted. */ - private Short segment; + private Integer segment; /** * Dir (0,1) indicates the direction (0 for Eastbound and 1 for Westbound). */ @@ -51,7 +51,7 @@ public SegmentIdentifier() { * @param segment the segment id * @param direction the direction */ - public SegmentIdentifier(Integer xWay, Short segment, Short direction) { + public SegmentIdentifier(Integer xWay, Integer segment, Short direction) { assert (xWay != null); assert (segment != null); assert (direction != null); @@ -107,14 +107,14 @@ public void setXWay(Integer xway) { * @return the segment number */ @Override - public Short getSegment() { + public int getSegment() { return this.segment; } /** * Sets the segment number. */ - public void setSegment(Short segment) { + public void setSegment(Integer segment) { this.segment = segment; } diff --git a/application/src/main/java/common/param/lr/LREvent.java b/application/src/main/java/common/param/lr/LREvent.java index adbe99f4c..6539f0441 100644 --- a/application/src/main/java/common/param/lr/LREvent.java +++ b/application/src/main/java/common/param/lr/LREvent.java @@ -1,12 +1,13 @@ package common.param.lr; import common.datatype.PositionReport; +import common.param.TxnEvent; import storage.SchemaRecordRef; /** * Currently only consider position events. */ -public class LREvent { +public class LREvent extends TxnEvent { private final int tthread; private final long bid; private final PositionReport posreport;//input_event associated common.meta data. @@ -25,6 +26,7 @@ public class LREvent { * @param bid */ public LREvent(PositionReport posreport, int tthread, long bid) { + super(bid); this.posreport = posreport; this.tthread = tthread; // vsreport = vehicleSpeedTuple; @@ -56,4 +58,9 @@ public long getBid() { public long[] getBid_array() { return new long[0]; } + + @Override + public LREvent cloneEvent() { + return new LREvent(this.posreport,tthread,bid); + } } \ No newline at end of file diff --git a/application/src/main/java/common/param/ob/AlertEvent.java b/application/src/main/java/common/param/ob/AlertEvent.java index be8697d12..11f3202b7 100644 --- a/application/src/main/java/common/param/ob/AlertEvent.java +++ b/application/src/main/java/common/param/ob/AlertEvent.java @@ -22,8 +22,8 @@ public class AlertEvent extends TxnEvent { public AlertEvent( int num_access, int[] itemId, SplittableRandom rnd, - int partition_id, long[] bid_array, long bid, int number_of_partitions) { - super(bid, partition_id, bid_array, number_of_partitions); + int partition_id, String bid_array, long bid, int number_of_partitions,String partition_index) { + super(bid, partition_id, bid_array,partition_index, number_of_partitions); this.num_access = num_access; record_refs = new SchemaRecordRef[num_access]; for (int i = 0; i < num_access; i++) { @@ -33,6 +33,7 @@ public AlertEvent( setValues(num_access, rnd); } + public AlertEvent(int bid, String bid_array, int partition_id, int number_of_partitions, int num_access, String key_array, String alert_array) { super(bid, partition_id, bid_array, number_of_partitions); @@ -94,4 +95,9 @@ public String toString() { + ", ask_price=" + Arrays.toString(ask_price) + '}'; } + + @Override + public AlertEvent cloneEvent() { + return new AlertEvent((int) bid,Arrays.toString(bid_array),pid,number_of_partitions,num_access,Arrays.toString(itemId),Arrays.toString(ask_price)); + } } diff --git a/application/src/main/java/common/param/ob/BuyingEvent.java b/application/src/main/java/common/param/ob/BuyingEvent.java index bfb121348..c10c0766d 100644 --- a/application/src/main/java/common/param/ob/BuyingEvent.java +++ b/application/src/main/java/common/param/ob/BuyingEvent.java @@ -25,7 +25,7 @@ public class BuyingEvent extends TxnEvent { /** * Creates a new BuyingEvent. */ - public BuyingEvent(int[] itemId, SplittableRandom rnd, int partition_id, long[] bid_array, long bid, int number_of_partitions) { + public BuyingEvent(int[] itemId, int partition_id, long[] bid_array, long bid, int number_of_partitions) { super(bid, partition_id, bid_array, number_of_partitions); this.itemId = itemId; record_refs = new SchemaRecordRef[NUM_ACCESSES_PER_BUY]; @@ -34,9 +34,25 @@ public BuyingEvent(int[] itemId, SplittableRandom rnd, int partition_id, long[] } bid_price = new long[NUM_ACCESSES_PER_BUY]; bid_qty = new long[NUM_ACCESSES_PER_BUY]; - setValues(rnd); + setValues(false); } + /** + * Creates a new BuyingEvent. + */ + public BuyingEvent(int[] itemId, int partition_id, String bid_array, long bid, int number_of_partitions,String partition_index,boolean isAbort) { + super(bid, partition_id, bid_array,partition_index, number_of_partitions); + this.itemId = itemId; + record_refs = new SchemaRecordRef[NUM_ACCESSES_PER_BUY]; + for (int i = 0; i < NUM_ACCESSES_PER_BUY; i++) { + record_refs[i] = new SchemaRecordRef(); + } + bid_price = new long[NUM_ACCESSES_PER_BUY]; + bid_qty = new long[NUM_ACCESSES_PER_BUY]; + setValues(isAbort); + } + + /** * Loading a BuyingEvent. */ @@ -106,14 +122,24 @@ public long[] getBidQty() { return bid_qty; } - private void set_values(int access_id, SplittableRandom rnd) { - bid_price[access_id] = rnd.nextLong(MAX_Price); - bid_qty[access_id] = rnd.nextLong(MAX_BUY_Transfer); + private void set_values(int access_id,boolean isAbort) { + if (isAbort) { + bid_price[access_id] = -1; + bid_qty[access_id] = 100000; + } else { + bid_price[access_id] = MAX_Price + 1; + bid_qty[access_id] = 1; + } } - public void setValues(SplittableRandom rnd) { + public void setValues(boolean isAbort) { for (int access_id = 0; access_id < NUM_ACCESSES_PER_BUY; ++access_id) { - set_values(access_id, rnd); + set_values(access_id,isAbort); } } + + @Override + public BuyingEvent cloneEvent() { + return new BuyingEvent((int) bid,Arrays.toString(bid_array),pid,number_of_partitions,Arrays.toString(itemId),Arrays.toString(bid_price),Arrays.toString(bid_qty)); + } } diff --git a/application/src/main/java/common/param/ob/ToppingEvent.java b/application/src/main/java/common/param/ob/ToppingEvent.java index abd8c1845..fe099ab25 100644 --- a/application/src/main/java/common/param/ob/ToppingEvent.java +++ b/application/src/main/java/common/param/ob/ToppingEvent.java @@ -22,8 +22,8 @@ public class ToppingEvent extends TxnEvent { public ToppingEvent( int num_access, int[] itemId, SplittableRandom rnd, - int partition_id, long[] bid_array, long bid, int number_of_partitions) { - super(bid, partition_id, bid_array, number_of_partitions); + int partition_id, String bid_array, long bid, int number_of_partitions,String partition_index) { + super(bid, partition_id, bid_array,partition_index, number_of_partitions); record_refs = new SchemaRecordRef[num_access]; this.num_access = num_access; for (int i = 0; i < num_access; i++) { @@ -103,4 +103,9 @@ public String toString() { + ", itemTopUp=" + Arrays.toString(itemTopUp) + '}'; } + + @Override + public ToppingEvent cloneEvent() { + return new ToppingEvent((int) bid, Arrays.toString(bid_array),pid,number_of_partitions,num_access,Arrays.toString(itemId),Arrays.toString(itemTopUp)); + } } diff --git a/application/src/main/java/common/sink/MeasureSink.java b/application/src/main/java/common/sink/MeasureSink.java index 8f0df27c2..c2ed47ba6 100644 --- a/application/src/main/java/common/sink/MeasureSink.java +++ b/application/src/main/java/common/sink/MeasureSink.java @@ -108,6 +108,28 @@ public void initialize(int task_Id_InGroup, int thisTaskId, ExecutionGraph graph config.getInt("Transaction_Length"), AppConfig.isCyclic, config.getInt("complexity")); + } else if (config.getString("common").equals("OnlineBiding")){ + directory = String.format(statsFolderPattern, + config.getString("common"), scheduler, tthread, totalEvents, + config.getInt("NUM_ITEMS"), + config.getInt("NUM_ACCESS"), + config.getInt("State_Access_Skewness"), + config.getInt("Ratio_of_Overlapped_Keys"), + config.getInt("Ratio_of_Transaction_Aborts"), + config.getInt("Transaction_Length"), + AppConfig.isCyclic, + config.getInt("complexity")); + } else if (config.getString("common").equals("TollProcessing")){ + directory = String.format(statsFolderPattern, + config.getString("common"), scheduler, tthread, totalEvents, + config.getInt("NUM_ITEMS"), + config.getInt("NUM_ACCESS"), + config.getInt("State_Access_Skewness"), + config.getInt("Ratio_of_Overlapped_Keys"), + config.getInt("Ratio_of_Transaction_Aborts"), + config.getInt("Transaction_Length"), + AppConfig.isCyclic, + config.getInt("complexity")); } else { throw new UnsupportedOperationException(); } diff --git a/application/src/main/java/common/topology/transactional/OnlineBiding.java b/application/src/main/java/common/topology/transactional/OnlineBiding.java index 93e03d6b1..82eca30df 100644 --- a/application/src/main/java/common/topology/transactional/OnlineBiding.java +++ b/application/src/main/java/common/topology/transactional/OnlineBiding.java @@ -38,8 +38,9 @@ public OnlineBiding(String topologyName, Configuration config) { public TableInitilizer initializeDB(SpinLock[] spinlock_) { double theta = config.getDouble("theta", 1); int tthread = config.getInt("tthread"); - setPartition_interval((int) (Math.ceil(NUM_ITEMS / (double) tthread)), tthread); - TableInitilizer ini = new OBInitializer(db, theta, tthread, config); + int numberOfStates = config.getInt("NUM_ITEMS"); + setPartition_interval((int) (Math.ceil(numberOfStates / (double) tthread)), tthread); + TableInitilizer ini = new OBInitializer(db,numberOfStates, theta, tthread, config); ini.creates_Table(config); if (config.getBoolean("partition", false)) { for (int i = 0; i < tthread; i++) diff --git a/application/src/main/java/common/topology/transactional/TollProcessing.java b/application/src/main/java/common/topology/transactional/TollProcessing.java index 8ddfea675..3d873a4e6 100644 --- a/application/src/main/java/common/topology/transactional/TollProcessing.java +++ b/application/src/main/java/common/topology/transactional/TollProcessing.java @@ -45,8 +45,9 @@ public void initialize() { public TableInitilizer initializeDB(SpinLock[] spinlock_) { double theta = config.getDouble("theta", 1); int tthread = config.getInt("tthread"); - setPartition_interval((int) (Math.ceil(NUM_SEGMENTS / (double) tthread)), tthread); - TableInitilizer ini = new TPInitializer(db, theta, tthread, config); + int numberOfStates = config.getInt("NUM_ITEMS"); + setPartition_interval((int) (Math.ceil(numberOfStates / (double) tthread)), tthread); + TableInitilizer ini = new TPInitializer(db,numberOfStates, theta, tthread, config); ini.creates_Table(config); if (config.getBoolean("partition", false)) { for (int i = 0; i < tthread; i++) diff --git a/application/src/main/java/common/topology/transactional/initializer/OBInitializer.java b/application/src/main/java/common/topology/transactional/initializer/OBInitializer.java index 0ee2f7575..00a0d30b8 100644 --- a/application/src/main/java/common/topology/transactional/initializer/OBInitializer.java +++ b/application/src/main/java/common/topology/transactional/initializer/OBInitializer.java @@ -1,10 +1,15 @@ package common.topology.transactional.initializer; +import benchmark.DataHolder; +import benchmark.datagenerator.DataGenerator; +import benchmark.datagenerator.DataGeneratorConfig; +import benchmark.datagenerator.apps.OB.OBTPGDynamicDataGenerator; +import benchmark.dynamicWorkloadGenerator.DynamicDataGeneratorConfig; import common.collections.Configuration; import common.collections.OsUtils; +import common.param.TxnEvent; import common.param.ob.AlertEvent; import common.param.ob.BuyingEvent; -import common.param.ob.OBParam; import common.param.ob.ToppingEvent; import db.Database; import db.DatabaseException; @@ -19,11 +24,13 @@ import storage.datatype.LongDataBox; import storage.table.RecordSchema; import transaction.TableInitilizer; +import utils.AppConfig; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; +import javax.xml.bind.DatatypeConverter; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.*; import static common.CONTROL.enable_log; @@ -32,7 +39,6 @@ import static common.constants.OnlineBidingSystemConstants.Constant.*; import static profiler.Metrics.NUM_ITEMS; import static transaction.State.configure_store; -import static utils.PartitionHelper.getPartition_interval; //import static xerial.jnuma.Numa.setLocalAlloc; public class OBInitializer extends TableInitilizer { @@ -40,19 +46,66 @@ public class OBInitializer extends TableInitilizer { //triple decisions protected int[] triple_decision = new int[]{0, 0, 0, 0, 0, 0, 1, 2};//6:1:1 buy, alert, topping_handle. private int i = 0; + private final int numberOfStates; + private String dataRootPath; + private DataGenerator dataGenerator; + private final DataGeneratorConfig dataConfig; + private final int partitionOffset; + private final int NUM_ACCESS; + private final int Transaction_Length; + private SplittableRandom rnd = new SplittableRandom(); - public OBInitializer(Database db, double theta, int tthread, Configuration config) { + public OBInitializer(Database db, int numberOfStates,double theta, int tthread, Configuration config) { super(db, theta, tthread, config); + floor_interval = (int) Math.floor(numberOfStates / (double) tthread);//NUM_ITEMS / tthread; + this.dataRootPath = config.getString("rootFilePath"); + this.partitionOffset = numberOfStates / tthread; + this.NUM_ACCESS = config.getInt("NUM_ACCESS"); + this.Transaction_Length = config.getInt("Transaction_Length"); + this.numberOfStates = numberOfStates; configure_store(theta, tthread, NUM_ITEMS); + createTPGGenerator(config); + dataConfig = dataGenerator.getDataConfig(); + } + + protected void createTPGGenerator(Configuration config) { + DynamicDataGeneratorConfig dynamicDataGeneratorConfig = new DynamicDataGeneratorConfig(); + dynamicDataGeneratorConfig.initialize(config); + configurePath(dynamicDataGeneratorConfig); + dataGenerator = new OBTPGDynamicDataGenerator(dynamicDataGeneratorConfig); + } + + private void configurePath(DynamicDataGeneratorConfig dynamicDataGeneratorConfig) { + MessageDigest digest; + String subFolder = null; + try { + digest = MessageDigest.getInstance("SHA-256"); + byte[] bytes; + bytes = digest.digest(String.format("%d_%d_%d_%s_%s_%s", + dynamicDataGeneratorConfig.getTotalThreads(), + dynamicDataGeneratorConfig.getTotalEvents(), + dynamicDataGeneratorConfig.getnKeyStates(), + dynamicDataGeneratorConfig.getType(), + dynamicDataGeneratorConfig.getApp(), + AppConfig.isCyclic) + .getBytes(StandardCharsets.UTF_8)); + subFolder = OsUtils.osWrapperPostFix( + DatatypeConverter.printHexBinary(bytes)); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } + dynamicDataGeneratorConfig.setRootPath(dynamicDataGeneratorConfig.getRootPath() + OsUtils.OS_wrapper(subFolder)); + dynamicDataGeneratorConfig.setIdsPath(dynamicDataGeneratorConfig.getIdsPath() + OsUtils.OS_wrapper(subFolder)); + this.dataRootPath += OsUtils.OS_wrapper(subFolder); } @Override public void loadDB(int thread_id, int NUMTasks) { - int partition_interval = getPartition_interval(); + int partition_interval = (int) Math.ceil(numberOfStates / (double) NUMTasks);; int left_bound = thread_id * partition_interval; int right_bound; if (thread_id == NUMTasks - 1) {//last executor need to handle left-over - right_bound = NUM_ITEMS; + right_bound = numberOfStates; } else { right_bound = (thread_id + 1) * partition_interval; } @@ -65,11 +118,11 @@ public void loadDB(int thread_id, int NUMTasks) { @Override public void loadDB(int thread_id, SpinLock[] spinlock, int NUMTasks) { - int partition_interval = getPartition_interval(); + int partition_interval = (int) Math.ceil(numberOfStates / (double) NUMTasks); int left_bound = thread_id * partition_interval; int right_bound; if (thread_id == NUMTasks - 1) {//last executor need to handle left-over - right_bound = NUM_ITEMS; + right_bound = numberOfStates; } else { right_bound = (thread_id + 1) * partition_interval; } @@ -134,83 +187,92 @@ private RecordSchema Goods() { return new RecordSchema(fieldNames, dataBoxes); } - /** - * OB - * - * @param partition_id - * @param bid_array - * @param number_of_partitions - * @param bid - * @param rnd - * @return - */ - protected BuyingEvent randomBuyEvents(int partition_id, long[] bid_array, int number_of_partitions, long bid, SplittableRandom rnd) { - int pid = partition_id; - OBParam param = new OBParam(NUM_ACCESSES_PER_BUY); - Set keys = new HashSet(); - int access_per_partition = (int) Math.ceil(NUM_ACCESSES_PER_BUY / (double) number_of_partitions); - int counter = 0; - randomkeys(pid, param, keys, access_per_partition, counter, NUM_ACCESSES_PER_BUY); - assert !enable_states_partition || verify(keys, partition_id, number_of_partitions); - return new BuyingEvent(param.keys(), rnd, partition_id, bid_array, bid, number_of_partitions); - } - - protected AlertEvent randomAlertEvents(int partition_id, long[] bid_array, int number_of_partitions, long bid, SplittableRandom rnd) { - int pid = partition_id; - int num_access = rnd.nextInt(NUM_ACCESSES_PER_ALERT) + 5; - OBParam param = new OBParam(num_access); - Set keys = new HashSet(); - int access_per_partition = (int) Math.ceil(num_access / (double) number_of_partitions); - int counter = 0; - randomkeys(pid, param, keys, access_per_partition, counter, num_access); - assert verify(keys, partition_id, number_of_partitions); - return new AlertEvent( - num_access, - param.keys(), - rnd, - partition_id, bid_array, bid, number_of_partitions - ); - } - - protected ToppingEvent randomToppingEvents(int partition_id, long[] bid_array, int number_of_partitions, long bid, SplittableRandom rnd) { - int pid = partition_id; - int num_access = rnd.nextInt(NUM_ACCESSES_PER_TOP) + 5; - OBParam param = new OBParam(num_access); - Set keys = new HashSet(); - int access_per_partition = (int) Math.ceil(num_access / (double) number_of_partitions); - int counter = 0; - randomkeys(pid, param, keys, access_per_partition, counter, num_access); - assert verify(keys, partition_id, number_of_partitions); - return new ToppingEvent( - num_access, - param.keys(), - rnd, - partition_id, bid_array, bid, number_of_partitions - ); - } - - protected int next_decision3() { - int rt = triple_decision[i]; - i++; - if (i == 8) - i = 0; - return rt; - } - @Override public boolean Generate() throws IOException { -// String event_path = Event_Path -// + OsUtils.OS_wrapper("enable_states_partition=" + enable_states_partition); -// return !Files.notExists(Paths.get(event_path + OsUtils.OS_wrapper(file))); - throw new UnsupportedOperationException(); + String folder = dataRootPath; + File file = new File(folder); + if (file.exists()) { + if (enable_log) LOG.info("Data already exists.. skipping data generation..."); + return false; + } + file.mkdirs(); + dataGenerator.generateStream();//prepare input events. + if (enable_log) LOG.info(String.format("Data Generator will dump data at %s.", dataRootPath)); + dataGenerator.dumpGeneratedDataToFile(); + if (enable_log) LOG.info("Data Generation is done..."); + dataGenerator.clearDataStructures(); + return true; } @Override protected void Load() throws IOException { - throw new UnsupportedOperationException(); + int totalEvents = dataConfig.getTotalEvents(); + boolean shufflingActive = dataConfig.getShufflingActive(); + String folder = dataConfig.getRootPath(); + File file = new File(folder + "events.txt"); + long[] p_bids = new long[tthread]; + if (file.exists()) { + if (enable_log) LOG.info("Reading transfer events..."); + BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file))); + loadOnlineBiddingEvents(reader, totalEvents, shufflingActive, p_bids); + reader.close(); + } + } + private void loadOnlineBiddingEvents(BufferedReader reader, int totalEvents, boolean shufflingActive, long[] p_bids) throws IOException { + String txn = reader.readLine(); + int count = 0; + while (txn != null) { + TxnEvent event; + String[] split = txn.split(","); + if (split.length > 3) { + int npid = (int) (Long.parseLong(split[1]) / partitionOffset); + int keyLength = NUM_ACCESS*Transaction_Length; + HashMap pids = new HashMap<>(); + int[] keys = new int[keyLength]; + for (int i = 1; i < keyLength+1; i++) { + keys[i-1] = Integer.parseInt(split[i]); + pids.put((int) (keys[i-1] / partitionOffset), 0); + } + if (split[split.length-1].equals("1")) { + event = new AlertEvent(keyLength, keys, rnd, npid, Arrays.toString(p_bids), Integer.parseInt(split[0]), pids.size(),Arrays.toString(pids.keySet().toArray(new Integer[0]))); + } else { + event = new ToppingEvent(keyLength, keys, rnd, npid, Arrays.toString(p_bids), Integer.parseInt(split[0]), pids.size(),Arrays.toString(pids.keySet().toArray(new Integer[0]))); + } + } else { + int npid = (int) (Long.parseLong(split[1]) / partitionOffset); + int[] keys = new int[NUM_ACCESSES_PER_BUY]; + HashMap pids = new HashMap<>(); + for (int i = 1; i < NUM_ACCESSES_PER_BUY+1; i++) { + keys[i-1] = Integer.parseInt(split[i]); + pids.put((int) (keys[i-1] / partitionOffset), 0); + } + if (split[split.length-1].equals("true")) { + event = new BuyingEvent(keys,npid, Arrays.toString(p_bids),Integer.parseInt(split[0]),pids.size(),Arrays.toString(pids.keySet().toArray(new Integer[0])),true); + } else { + event = new BuyingEvent(keys,npid, Arrays.toString(p_bids),Integer.parseInt(split[0]),pids.size(),Arrays.toString(pids.keySet().toArray(new Integer[0])),false); + } + } + DataHolder.events.add(event); + if (enable_log) LOG.debug(String.format("%d deposit read...", count)); + txn = reader.readLine(); + } + if (enable_log) LOG.info("Done reading transfer events..."); + if (shufflingActive) { + shuffleEvents(DataHolder.events, totalEvents); + } + } + private void shuffleEvents(ArrayList txnEvents, int totalEvents) { + Random random = new Random(); + int index; + TxnEvent temp; + for (int i = totalEvents - 1; i > 0; i--) { + index = random.nextInt(i + 1); + temp = txnEvents.get(index); + txnEvents.set(index, txnEvents.get(i)); + txnEvents.set(i, temp); + } } - @Override public void store(String file_name) throws IOException { diff --git a/application/src/main/java/common/topology/transactional/initializer/TPInitializer.java b/application/src/main/java/common/topology/transactional/initializer/TPInitializer.java index c002b8d12..8d6f06072 100644 --- a/application/src/main/java/common/topology/transactional/initializer/TPInitializer.java +++ b/application/src/main/java/common/topology/transactional/initializer/TPInitializer.java @@ -1,6 +1,16 @@ package common.topology.transactional.initializer; +import benchmark.DataHolder; +import benchmark.datagenerator.DataGenerator; +import benchmark.datagenerator.DataGeneratorConfig; +import benchmark.datagenerator.apps.TP.TPTPGDynamicDataGenerator; +import benchmark.dynamicWorkloadGenerator.DynamicDataGeneratorConfig; import common.collections.Configuration; +import common.collections.OsUtils; +import common.datatype.PositionReport; +import common.param.TxnEvent; +import common.param.lr.LREvent; +import common.tools.randomNumberGenerator; import db.Database; import db.DatabaseException; import lock.SpinLock; @@ -15,29 +25,81 @@ import storage.datatype.StringDataBox; import storage.table.RecordSchema; import transaction.TableInitilizer; +import utils.AppConfig; -import java.io.IOException; +import javax.xml.bind.DatatypeConverter; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; +import java.util.Random; import static common.CONTROL.enable_log; import static common.constants.TPConstants.Constant.NUM_SEGMENTS; +import static profiler.Metrics.NUM_ITEMS; +import static transaction.State.configure_store; import static utils.PartitionHelper.getPartition_interval; public class TPInitializer extends TableInitilizer { private static final Logger LOG = LoggerFactory.getLogger(TPInitializer.class); - - public TPInitializer(Database db, double theta, int tthread, Configuration config) { + private final int numberOfStates; + private String dataRootPath; + private DataGenerator dataGenerator; + private final DataGeneratorConfig dataConfig; + private final int partitionOffset; + private final int NUM_ACCESS; + private final int Transaction_Length; + public TPInitializer(Database db,int numberOfStates, double theta, int tthread, Configuration config) { super(db, theta, tthread, config); + floor_interval = (int) Math.floor(numberOfStates / (double) tthread);//NUM_ITEMS / tthread; + this.dataRootPath = config.getString("rootFilePath"); + this.partitionOffset = numberOfStates / tthread; + this.NUM_ACCESS = config.getInt("NUM_ACCESS"); + this.Transaction_Length = config.getInt("Transaction_Length"); + this.numberOfStates = numberOfStates; + configure_store(theta, tthread, numberOfStates); + createTPGGenerator(config); + dataConfig = dataGenerator.getDataConfig(); + } + protected void createTPGGenerator(Configuration config) { + DynamicDataGeneratorConfig dynamicDataGeneratorConfig = new DynamicDataGeneratorConfig(); + dynamicDataGeneratorConfig.initialize(config); + configurePath(dynamicDataGeneratorConfig); + dataGenerator = new TPTPGDynamicDataGenerator(dynamicDataGeneratorConfig); + } + private void configurePath(DynamicDataGeneratorConfig dynamicDataGeneratorConfig) { + MessageDigest digest; + String subFolder = null; + try { + digest = MessageDigest.getInstance("SHA-256"); + byte[] bytes; + bytes = digest.digest(String.format("%d_%d_%d_%s_%s_%s", + dynamicDataGeneratorConfig.getTotalThreads(), + dynamicDataGeneratorConfig.getTotalEvents(), + dynamicDataGeneratorConfig.getnKeyStates(), + dynamicDataGeneratorConfig.getType(), + dynamicDataGeneratorConfig.getApp(), + AppConfig.isCyclic) + .getBytes(StandardCharsets.UTF_8)); + subFolder = OsUtils.osWrapperPostFix( + DatatypeConverter.printHexBinary(bytes)); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } + dynamicDataGeneratorConfig.setRootPath(dynamicDataGeneratorConfig.getRootPath() + OsUtils.OS_wrapper(subFolder)); + dynamicDataGeneratorConfig.setIdsPath(dynamicDataGeneratorConfig.getIdsPath() + OsUtils.OS_wrapper(subFolder)); + this.dataRootPath += OsUtils.OS_wrapper(subFolder); } @Override public void loadDB(int thread_id, SpinLock[] spinlock, int NUMTasks) { - int partition_interval = getPartition_interval(); + int partition_interval = (int) Math.ceil(numberOfStates / (double) NUMTasks);; int left_bound = thread_id * partition_interval; int right_bound; if (thread_id == NUMTasks - 1) {//last executor need to handle left-over - right_bound = NUM_SEGMENTS; + right_bound = numberOfStates; } else { right_bound = (thread_id + 1) * partition_interval; } @@ -63,11 +125,11 @@ public void loadDB(SchedulerContext context, int thread_id, SpinLock[] spinlock, @Override public void loadDB(int thread_id, int NUMTasks) { - int partition_interval = getPartition_interval(); + int partition_interval = (int) Math.ceil(numberOfStates / (double) NUMTasks); int left_bound = thread_id * partition_interval; int right_bound; if (thread_id == NUMTasks - 1) {//last executor need to handle left-over - right_bound = NUM_SEGMENTS; + right_bound = numberOfStates; } else { right_bound = (thread_id + 1) * partition_interval; } @@ -150,13 +212,82 @@ private RecordSchema CntScheme() { @Override public boolean Generate() { + String folder = dataRootPath; + File file = new File(folder); + if (file.exists()) { + if (enable_log) LOG.info("Data already exists.. skipping data generation..."); + return false; + } + file.mkdirs(); + dataGenerator.generateStream();//prepare input events. + if (enable_log) LOG.info(String.format("Data Generator will dump data at %s.", dataRootPath)); + dataGenerator.dumpGeneratedDataToFile(); + if (enable_log) LOG.info("Data Generation is done..."); + dataGenerator.clearDataStructures(); return true; } @Override protected void Load() throws IOException { - throw new UnsupportedOperationException(); + int totalEvents = dataConfig.getTotalEvents(); + boolean shufflingActive = dataConfig.getShufflingActive(); + String folder = dataConfig.getRootPath(); + File file = new File(folder + "events.txt"); + long[] p_bids = new long[tthread]; + if (file.exists()) { + if (enable_log) LOG.info("Reading transfer events..."); + BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file))); + loadTollProcessingEvents(reader, totalEvents, shufflingActive, p_bids); + reader.close(); + } + } + private void loadTollProcessingEvents(BufferedReader reader, int totalEvents, boolean shufflingActive, long[] p_bids) throws IOException { + String txn = reader.readLine(); + int count = 0; + while (txn != null) { + LREvent event; + PositionReport positionReport; + String[] split = txn.split(","); + if (split[split.length-1].equals("true")) { + positionReport = new PositionReport((short) 0, + randomNumberGenerator.generate(1,100), + 200, + randomNumberGenerator.generate(1,4), + (short)randomNumberGenerator.generate(1,4), + (short)randomNumberGenerator.generate(1,1), + Integer.parseInt(split[1]), + randomNumberGenerator.generate(1,100)); + } else { + positionReport = new PositionReport((short) 0, + randomNumberGenerator.generate(1,100), + randomNumberGenerator.generate(60,180), + randomNumberGenerator.generate(1,4), + (short)randomNumberGenerator.generate(1,4), + (short)randomNumberGenerator.generate(1,1), + Integer.parseInt(split[1]), + randomNumberGenerator.generate(1,100)); + } + event = new LREvent(positionReport,dataConfig.getTotalThreads(),Long.parseLong(split[0])); + DataHolder.events.add(event); + if (enable_log) LOG.debug(String.format("%d deposit read...", count)); + txn = reader.readLine(); + } + if (enable_log) LOG.info("Done reading transfer events..."); + if (shufflingActive) { + shuffleEvents(DataHolder.events, totalEvents); + } + } + private void shuffleEvents(ArrayList txnEvents, int totalEvents) { + Random random = new Random(); + int index; + TxnEvent temp; + for (int i = totalEvents - 1; i > 0; i--) { + index = random.nextInt(i + 1); + temp = txnEvents.get(index); + txnEvents.set(index, txnEvents.get(i)); + txnEvents.set(i, temp); + } } @Override @@ -172,5 +303,10 @@ public void creates_Table(Configuration config) { db.createTable(s, "segment_speed"); RecordSchema b = CntScheme(); db.createTable(b, "segment_cnt"); + try { + prepare_input_events(config.getInt("totalEvents")); + } catch (IOException e) { + e.printStackTrace(); + } } } diff --git a/application/src/main/java/runners/MorphStreamRunner.java b/application/src/main/java/runners/MorphStreamRunner.java index 3e8b6de8a..93acc10cc 100644 --- a/application/src/main/java/runners/MorphStreamRunner.java +++ b/application/src/main/java/runners/MorphStreamRunner.java @@ -127,13 +127,13 @@ private void LoadConfiguration() { break; } case "OnlineBiding": { - config.put("app", 2); + config.put("app", 3); int threads = Math.max(1, (int) Math.floor((tthread))); config.put(OB_THREADS, threads); break; } case "TollProcessing": { - config.put("app", 3); + config.put("app", 2); int threads = Math.max(1, (int) Math.floor((tthread))); config.put(Executor_Threads, threads); break; @@ -269,9 +269,30 @@ public void run() throws InterruptedException { config.getInt("Transaction_Length"), AppConfig.isCyclic, config.getInt("complexity")); - } else { + } else if (config.getString("common").equals("OnlineBiding")){ + statsFolderPath = String.format(statsFolderPattern, + config.getString("common"), scheduler, tthread, totalEvents, + config.getInt("NUM_ITEMS"), + config.getInt("NUM_ACCESS"), + config.getInt("State_Access_Skewness"), + config.getInt("Ratio_of_Overlapped_Keys"), + config.getInt("Ratio_of_Transaction_Aborts"), + config.getInt("Transaction_Length"), + AppConfig.isCyclic, + config.getInt("complexity")); + } else if (config.getString("common").equals("TollProcessing")){ + statsFolderPath = String.format(statsFolderPattern, + config.getString("common"), scheduler, tthread, totalEvents, + config.getInt("NUM_ITEMS"), + config.getInt("NUM_ACCESS"), + config.getInt("State_Access_Skewness"), + config.getInt("Ratio_of_Overlapped_Keys"), + config.getInt("Ratio_of_Transaction_Aborts"), + config.getInt("Transaction_Length"), + AppConfig.isCyclic, + config.getInt("complexity")); + } else throw new UnsupportedOperationException(); - } File file = new File(statsFolderPath); log.info("Dumping stats to..."); log.info(String.valueOf(file.getAbsoluteFile())); diff --git a/common/src/main/java/common/Runner.java b/common/src/main/java/common/Runner.java index d2c47f2fe..1cad5b4c5 100644 --- a/common/src/main/java/common/Runner.java +++ b/common/src/main/java/common/Runner.java @@ -19,8 +19,10 @@ public abstract class Runner implements IRunner { * Workload Specific Parameters. */ @Parameter(names = {"-a", "--app"}, description = "The application to be executed") - public String application = "StreamLedger"; + //public String application = "StreamLedger"; //public String application = "GrepSum"; + //public String application = "OnlineBiding"; + public String application = "TollProcessing"; @Parameter(names = {"-t", "--topology-name"}, required = false, description = "The name of the application") public String topologyName; @Parameter(names = {"--COMPUTE_COMPLEXITY"}, description = "COMPUTE_COMPLEXITY per event") @@ -65,7 +67,7 @@ public abstract class Runner implements IRunner { @Parameter(names = {"--partition"}, description = "Partitioning database. It must be enabled for S-Store scheme and it is optional for TStream scheme.") public boolean enable_partition = false; @Parameter(names = {"--scheduler"}, description = "Scheduler for TStream.") - public String scheduler = "OG_BFS"; + public String scheduler = "OP_BFS"; // public String scheduler = "OG_BFS_A"; // public String scheduler = "OG_DFS"; // public String scheduler = "OG_DFS_A"; @@ -135,11 +137,14 @@ public abstract class Runner implements IRunner { public String generator = "TPGGenerator"; // public String generator = "OCGenerator"; @Parameter(names = {"--totalEvents"}, description = "Total number of events to process.") - public int totalEvents = 100000; + public int totalEvents = 10000; @Parameter(names = {"--deposit_ratio"}, description = "Ratio of deposit for SL.") public Integer Ratio_Of_Deposit = 25; + @Parameter(names = {"--buying_ratio"}, description = "Ratio of buying for OB.") + public Integer Ratio_Of_Buying = 25; + @Parameter(names = {"--key_skewness"}, description = "State access skewness.") public Integer State_Access_Skewness = 0; @@ -156,7 +161,7 @@ public abstract class Runner implements IRunner { public Integer numberOfDLevels = 1024; @Parameter(names = {"--isCyclic"}, description = "isCyclic of generated OC.") - public int isCyclic = 1; + public int isCyclic = 0; @Parameter(names = {"--complexity"}, description = "Dummy UDF complexity for state access process.") public Integer complexity = 100000; @@ -188,7 +193,11 @@ public void initializeCfg(HashMap config) { config.put("fanoutDist", fanoutDist); config.put("idGenType", idGenType); - config.put("Ratio_Of_Deposit", Ratio_Of_Deposit); + if (application.equals("OnlineBiding")){ + config.put("Ratio_Of_Buying", Ratio_Of_Buying); + }else { + config.put("Ratio_Of_Deposit", Ratio_Of_Deposit); + } config.put("State_Access_Skewness", State_Access_Skewness); config.put("Ratio_of_Overlapped_Keys", Ratio_of_Overlapped_Keys); config.put("Ratio_of_Transaction_Aborts", Ratio_of_Transaction_Aborts); diff --git a/common/src/main/resources/config/TollProcessing.properties b/common/src/main/resources/config/TollProcessing.properties index 7b13c2575..7602857ce 100644 --- a/common/src/main/resources/config/TollProcessing.properties +++ b/common/src/main/resources/config/TollProcessing.properties @@ -1,4 +1,4 @@ -tptxn.spout.class=common.combo.TPCombo +tptxn.spout.class=combo.TPCombo batch_size.tptxn.spout.class=common.spout.BatchMemFileSpout tptxn.spout.parser=common.parser.StringParser #tptxn.sink.class=applications.sink.ConsoleSink diff --git a/pom.xml b/pom.xml index 2b3c23657..2345b66af 100644 --- a/pom.xml +++ b/pom.xml @@ -16,49 +16,6 @@ - - - - - maven-clean-plugin - 3.1.0 - - - - maven-resources-plugin - 3.0.2 - - - maven-compiler-plugin - 3.8.0 - - - maven-surefire-plugin - 2.22.1 - - - maven-jar-plugin - 3.0.2 - - - maven-install-plugin - 2.5.2 - - - maven-deploy-plugin - 2.8.2 - - - - maven-site-plugin - 3.7.1 - - - maven-project-info-reports-plugin - 3.0.0 - - - org.apache.maven.plugins diff --git a/state-engine/src/main/java/common/param/TxnEvent.java b/state-engine/src/main/java/common/param/TxnEvent.java index b9afeb00c..4c204b255 100644 --- a/state-engine/src/main/java/common/param/TxnEvent.java +++ b/state-engine/src/main/java/common/param/TxnEvent.java @@ -51,6 +51,12 @@ public TxnEvent(long bid, int partition_id, long[] bid_array, int number_of_part success = new int[1]; } + public TxnEvent(long bid) { + this.bid = bid; + this.pid = 0; + number_of_partitions = 1; + } + public long getBid() { return bid; } diff --git a/state-engine/src/main/java/scheduler/Request.java b/state-engine/src/main/java/scheduler/Request.java index 9d79036a6..b4235f387 100644 --- a/state-engine/src/main/java/scheduler/Request.java +++ b/state-engine/src/main/java/scheduler/Request.java @@ -23,6 +23,7 @@ public class Request { public final int[] success; public final int column_id; public final double[] enqueue_time; + public long value; public Request() { this(null, null, null); @@ -34,6 +35,16 @@ public Request(TxnContext txn_context, String table_name) { this(txn_context, accessType, table_name, null); } + //Write-only + public Request(TxnContext txn_context, + CommonMetaTypes.AccessType accessType, + String src_key, + String table_name, + TableRecord d_record, + long value) { + this(txn_context, accessType, table_name, src_key, null, d_record, null, null); + this.value = value; + } public Request(TxnContext txn_context, CommonMetaTypes.AccessType accessType, diff --git a/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java b/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java index 303e77c0b..e35c4bbee 100644 --- a/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java @@ -14,6 +14,9 @@ import storage.SchemaRecord; import storage.TableRecord; import storage.datatype.DataBox; +import storage.datatype.DoubleDataBox; +import storage.datatype.IntDataBox; +import transaction.function.AVG; import transaction.function.DEC; import transaction.function.INC; import transaction.function.SUM; @@ -22,6 +25,7 @@ import utils.SOURCE_CONTROL; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import static content.common.CommonMetaTypes.AccessType.*; @@ -73,7 +77,7 @@ public void start_evaluation(Context context, long mark_ID, int num_events) { EXPLORE(context); // MeasureTools.END_SCHEDULE_EXPLORE_TIME_MEASURE(threadId); // MeasureTools.BEGIN_SCHEDULE_USEFUL_TIME_MEASURE(threadId); - PROCESS(context, mark_ID); + PROCESS(context, mark_ID); // MeasureTools.END_SCHEDULE_USEFUL_TIME_MEASURE(threadId); // MeasureTools.END_SCHEDULE_EXPLORE_TIME_MEASURE(threadId); } while (!FINISHED(context)); @@ -195,12 +199,35 @@ public void execute(Operation operation, long mark_ID, boolean clean) { } } else if (operation.accessType.equals(READ_WRITE_COND)) { success = operation.success[0]; - Transfer_Fun(operation, mark_ID, clean); + if (this.tpg.getApp() == 1) {//SL + Transfer_Fun(operation, mark_ID, clean); + } else {//OB + AppConfig.randomDelay(); + List d_record = operation.condition_records[0].content_.ReadAccess(operation.bid, mark_ID, clean, operation.accessType).getValues(); + long askPrice = d_record.get(1).getLong();//price + long left_qty = d_record.get(2).getLong();//available qty; + long bidPrice = operation.condition.arg1; + long bid_qty = operation.condition.arg2; + if (bidPrice > askPrice || bid_qty < left_qty) { + d_record.get(2).setLong(left_qty - operation.function.delta_long);//new quantity. + operation.success[0]++; + } + } if (operation.success[0] == success) { operation.isFailed = true; } } else if (operation.accessType.equals(READ_WRITE)) { - Depo_Fun(operation, mark_ID, clean); + if (this.tpg.getApp() == 1) { //SL + Depo_Fun(operation, mark_ID, clean); + } else { + AppConfig.randomDelay(); + SchemaRecord srcRecord = operation.s_record.content_.ReadAccess(operation.bid,mark_ID,clean,operation.accessType); + List values = srcRecord.getValues(); + if (operation.function instanceof INC) { + values.get(2).setLong(values.get(2).getLong() + operation.function.delta_long); + } else + throw new UnsupportedOperationException(); + } } else if (operation.accessType.equals(READ_WRITE_COND_READN)) { success = operation.success[0]; GrepSum_Fun(operation, mark_ID, clean); @@ -210,6 +237,30 @@ public void execute(Operation operation, long mark_ID, boolean clean) { if (operation.success[0] == success) { operation.isFailed = true; } + } else if (operation.accessType.equals(READ_WRITE_READ)) { + //TODO: implement the operation + assert operation.record_ref != null; + AppConfig.randomDelay(); + List srcRecord = operation.s_record.record_.getValues(); + if (operation.function instanceof AVG) { + double latestAvgSpeeds = srcRecord.get(1).getDouble(); + double lav; + if (latestAvgSpeeds == 0) {//not initialized + lav = operation.function.delta_double; + } else + lav = (latestAvgSpeeds + operation.function.delta_double) / 2; + + srcRecord.get(1).setDouble(lav);//write to state. + operation.record_ref.setRecord(new SchemaRecord(new DoubleDataBox(lav)));//return updated record. + } else { + HashSet cnt_segment = srcRecord.get(1).getHashSet(); + cnt_segment.add(operation.function.delta_int);//update hashset; updated state also. TODO: be careful of this. + operation.record_ref.setRecord(new SchemaRecord(new IntDataBox(cnt_segment.size())));//return updated record. + } + } else if (operation.accessType.equals(WRITE_ONLY)) { + //OB-Alert + AppConfig.randomDelay(); + operation.d_record.record_.getValues().get(1).setLong(operation.value); } else { throw new UnsupportedOperationException(); } @@ -330,7 +381,7 @@ public void TxnSubmitFinished(Context context) { Operation headerOperation = null; Operation set_op; for (Request request : context.requests) { - set_op = constructOp(operationGraph, request); + set_op = constructOp(operationGraph, request); if (txnOpId == 0) headerOperation = set_op; // addOperation an operation id for the operation for the purpose of temporal dependency construction @@ -347,6 +398,11 @@ private Operation constructOp(List operationGraph, Request request) { Operation set_op; Context targetContext = getTargetContext(request.src_key); switch (request.accessType) { + case WRITE_ONLY: + set_op = new Operation(request.src_key, null, request.table_name, null, null, null, + null, request.txn_context, request.accessType, null, request.d_record, bid, targetContext); + set_op.value = request.value; + break; case READ_WRITE_COND: // they can use the same method for processing case READ_WRITE: set_op = new Operation(request.src_key, request.function, request.table_name, null, request.condition_records, request.condition, @@ -357,6 +413,10 @@ private Operation constructOp(List operationGraph, Request request) { set_op = new Operation(request.src_key, request.function, request.table_name, request.record_ref, request.condition_records, request.condition, request.success, request.txn_context, request.accessType, request.d_record, request.d_record, bid, targetContext); break; + case READ_WRITE_READ: + set_op = new Operation(request.src_key, request.function, request.table_name, request.record_ref, null, null, + null, request.txn_context, request.accessType, request.d_record, request.d_record, bid, targetContext); + break; default: throw new RuntimeException("Unexpected operation"); } diff --git a/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java b/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java index be0394d95..aa57b55a1 100644 --- a/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java @@ -14,12 +14,16 @@ import storage.SchemaRecord; import storage.TableRecord; import storage.datatype.DataBox; +import storage.datatype.DoubleDataBox; +import storage.datatype.IntDataBox; +import transaction.function.AVG; import transaction.function.DEC; import transaction.function.INC; import transaction.function.SUM; import utils.AppConfig; import utils.SOURCE_CONTROL; +import java.util.HashSet; import java.util.List; import static content.common.CommonMetaTypes.AccessType.*; @@ -88,13 +92,36 @@ public void execute(Operation operation, long mark_ID, boolean clean) { } } else if (operation.accessType.equals(READ_WRITE_COND)) { success = operation.success[0]; - Transfer_Fun(operation, mark_ID, clean); + if (this.tpg.getApp() == 1) {//SL + Transfer_Fun(operation, mark_ID, clean); + } else {//OB + AppConfig.randomDelay(); + List d_record = operation.condition_records[0].content_.ReadAccess(operation.bid, mark_ID, clean, operation.accessType).getValues(); + long askPrice = d_record.get(1).getLong();//price + long left_qty = d_record.get(2).getLong();//available qty; + long bidPrice = operation.condition.arg1; + long bid_qty = operation.condition.arg2; + if (bidPrice > askPrice || bid_qty < left_qty) { + d_record.get(2).setLong(left_qty - operation.function.delta_long);//new quantity. + operation.success[0]++; + } + } // operation success check, number of operation succeeded does not increase after execution if (operation.success[0] == success) { operation.isFailed = true; } } else if (operation.accessType.equals(READ_WRITE)) { - Depo_Fun(operation, mark_ID, clean); + if (this.tpg.getApp() == 1) { + Depo_Fun(operation, mark_ID, clean); + } else { + AppConfig.randomDelay(); + SchemaRecord srcRecord = operation.s_record.content_.ReadAccess(operation.bid,mark_ID,clean,operation.accessType); + List values = srcRecord.getValues(); + if (operation.function instanceof INC) { + values.get(2).setLong(values.get(2).getLong() + operation.function.delta_long); + } else + throw new UnsupportedOperationException(); + } } else if (operation.accessType.equals(READ_WRITE_COND_READN)) { success = operation.success[0]; GrepSum_Fun(operation, mark_ID, clean); @@ -106,6 +133,29 @@ public void execute(Operation operation, long mark_ID, boolean clean) { if (operation.success[0] == success) { operation.isFailed = true; } + } else if (operation.accessType.equals(WRITE_ONLY)) { + //OB-Alert + AppConfig.randomDelay(); + operation.d_record.record_.getValues().get(1).setLong(operation.value); + } else if (operation.accessType.equals(READ_WRITE_READ)){ + assert operation.record_ref != null; + AppConfig.randomDelay(); + List srcRecord = operation.s_record.record_.getValues(); + if (operation.function instanceof AVG) { + double latestAvgSpeeds = srcRecord.get(1).getDouble(); + double lav; + if (latestAvgSpeeds == 0) {//not initialized + lav = operation.function.delta_double; + } else + lav = (latestAvgSpeeds + operation.function.delta_double) / 2; + + srcRecord.get(1).setDouble(lav);//write to state. + operation.record_ref.setRecord(new SchemaRecord(new DoubleDataBox(lav)));//return updated record. + } else { + HashSet cnt_segment = srcRecord.get(1).getHashSet(); + cnt_segment.add(operation.function.delta_int);//update hashset; updated state also. TODO: be careful of this. + operation.record_ref.setRecord(new SchemaRecord(new IntDataBox(cnt_segment.size())));//return updated record. + } } else { throw new UnsupportedOperationException(); } @@ -236,6 +286,11 @@ public void TxnSubmitFinished(Context context) { long bid = request.txn_context.getBID(); Operation set_op; switch (request.accessType) { + case WRITE_ONLY: + set_op = new Operation(request.src_key, getTargetContext(request.src_key), request.table_name, request.txn_context, bid, request.accessType, + request.d_record, null, null, null, null); + set_op.value = request.value; + break; case READ_WRITE: // they can use the same method for processing case READ_WRITE_COND: set_op = new Operation(request.src_key, getTargetContext(request.src_key), request.table_name, request.txn_context, bid, request.accessType, diff --git a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java index 2c55949bd..8255a33a6 100644 --- a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java @@ -54,6 +54,11 @@ public void reset(Context context) { } else if (app == 1) { operationChains.get("accounts").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); operationChains.get("bookEntries").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); + } else if (app == 2 ) { + operationChains.get("segment_speed").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); + operationChains.get("segment_cnt").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); + } else if (app == 3) { + operationChains.get("goods").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); } // threadToOCs.get(context.thisThreadId).clear(); for (OperationChain oc : threadToOCs.get(context.thisThreadId)) { @@ -83,9 +88,11 @@ public TaskPrecedenceGraph(int totalThreads, int delta, int NUM_ITEMS, int app) } else if (app == 1) {//SL operationChains.put("accounts", new TableOCs<>(totalThreads)); operationChains.put("bookEntries", new TableOCs<>(totalThreads)); - } else if(app==2){//TP + } else if(app == 2) {//TP operationChains.put("segment_speed",new TableOCs<>(totalThreads)); operationChains.put("segment_cnt",new TableOCs<>(totalThreads)); + } else if (app == 3) { + operationChains.put("goods",new TableOCs<>(totalThreads)); } } @@ -117,13 +124,17 @@ public void setOCs(Context context) { operationChains.get("bookEntries").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, beOC); ocs.add(accOC); ocs.add(beOC); - }else if(app==2){ + } else if( app == 2) { OperationChain speedOC=context.createTask("segment_speed",_key,0); OperationChain cntOC=context.createTask("segment_cnt",_key,0); operationChains.get("segment_speed").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, speedOC); operationChains.get("segment_cnt").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, cntOC); ocs.add(speedOC); ocs.add(cntOC); + } else if (app == 3){ + OperationChain gsOC = context.createTask("goods", _key, 0); + operationChains.get("goods").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, gsOC); + ocs.add(gsOC); } } threadToOCs.put(context.thisThreadId, ocs); @@ -489,4 +500,8 @@ private void checkFD(OperationChain curOC, Operation op, String table_name, public Collection getContexts() { return threadToContextMap.values(); } + + public int getApp() { + return app; + } } diff --git a/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java index c69cd3f1a..9b5a09e12 100644 --- a/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java @@ -48,6 +48,11 @@ public void reset(Context context) { } else if (app == 1) { operationChains.get("accounts").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); operationChains.get("bookEntries").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); + } else if (app == 2) { + operationChains.get("segment_speed").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); + operationChains.get("segment_cnt").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); + } else if (app == 3) { + operationChains.get("goods").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); } // threadToOCs.get(context.thisThreadId).clear(); for (OperationChain oc : threadToOCs.get(context.thisThreadId)) { @@ -75,9 +80,11 @@ public TaskPrecedenceGraph(int totalThreads, int delta, int NUM_ITEMS, int app) } else if (app == 1) {//SL operationChains.put("accounts", new TableOCs(totalThreads)); operationChains.put("bookEntries", new TableOCs(totalThreads)); - } else if(app==2){//TP + } else if (app == 2){//TP operationChains.put("segment_speed",new TableOCs(totalThreads)); operationChains.put("segment_cnt",new TableOCs(totalThreads)); + } else if (app == 3) {//OB + operationChains.put("goods",new TableOCs(totalThreads)); } } @@ -109,13 +116,17 @@ public void setOCs(Context context) { operationChains.get("bookEntries").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, beOC); ocs.add(accOC); ocs.add(beOC); - } else if(app==2){ + } else if (app == 2) { OperationChain speedOC=context.createTask("segment_speed",_key); OperationChain cntOC=context.createTask("segment_cnt",_key); operationChains.get("segment_speed").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, speedOC); operationChains.get("segment_cnt").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, cntOC); ocs.add(speedOC); ocs.add(cntOC); + } else if (app ==3) { + OperationChain gsOC = context.createTask("goods", _key); + operationChains.get("goods").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, gsOC); + ocs.add(gsOC); } } threadToOCs.put(context.thisThreadId, ocs); @@ -316,4 +327,8 @@ private void checkFD(OperationChain curOC, Operation op, String table_name, curOC.checkPotentialFDChildrenOnNewArrival(op); } } + + public int getApp() { + return app; + } } diff --git a/state-engine/src/main/java/transaction/impl/TxnManagerDedicatedAsy.java b/state-engine/src/main/java/transaction/impl/TxnManagerDedicatedAsy.java index d3dd6311d..8c76e6c21 100644 --- a/state-engine/src/main/java/transaction/impl/TxnManagerDedicatedAsy.java +++ b/state-engine/src/main/java/transaction/impl/TxnManagerDedicatedAsy.java @@ -126,7 +126,7 @@ public boolean Asy_WriteRecord(TxnContext txn_context, String srcTable, String p AccessType accessType = AccessType.WRITE_ONLY; TableRecord t_record = storageManager_.getTable(srcTable).SelectKeyRecord(primary_key); if (t_record != null) { - return scheduler.SubmitRequest(context, new Request(txn_context, accessType, srcTable)); + return scheduler.SubmitRequest(context, new Request(txn_context, accessType,primary_key,srcTable,t_record,value)); } else { if (enable_log) log.info("No record is found:" + primary_key); return false; From 281a663e16e6e3cfb8f525ee9ef386f752a2255c Mon Sep 17 00:00:00 2001 From: curryzjj Date: Fri, 18 Mar 2022 16:52:14 +0800 Subject: [PATCH 12/15] Fix some typo --- .../apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java | 1 - .../apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java | 8 -------- .../datagenerator/apps/TP/TPTPGDynamicDataGenerator.java | 4 ++++ .../src/main/java/scheduler/impl/og/OGScheduler.java | 2 +- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java index d5cb67fdb..e619e35a3 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java @@ -1,7 +1,6 @@ package benchmark.datagenerator.apps.GS.TPGTxnGenerator; import benchmark.datagenerator.DataGenerator; -import benchmark.datagenerator.DataGeneratorConfig; import benchmark.datagenerator.Event; import benchmark.datagenerator.apps.GS.TPGTxnGenerator.Transaction.GSEvent; import common.tools.FastZipfGenerator; diff --git a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java index e2d6c6963..d716a0dde 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java +++ b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java @@ -26,12 +26,4 @@ public void initialize(Configuration config) { Ratio_of_Transaction_Aborts = config.getInt("Ratio_of_Transaction_Aborts", 0); Transaction_Length = config.getInt("Transaction_Length", 1); } - - public void setConfiguration(int num_access, int state_Access_Skewness, int ratio_of_Overlapped_Keys,int ratio_of_Transaction_Aborts, int transaction_Length){ - NUM_ACCESS = num_access; - State_Access_Skewness = state_Access_Skewness; - Ratio_of_Overlapped_Keys = ratio_of_Overlapped_Keys; - Ratio_of_Transaction_Aborts = ratio_of_Transaction_Aborts; - Transaction_Length = transaction_Length; - } } \ No newline at end of file diff --git a/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java index 9eff29c23..73b62df1b 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java @@ -18,6 +18,10 @@ import static common.CONTROL.enable_log; import static common.CONTROL.enable_states_partition; +/** + * Dynamic workload generator for toll processing + * Created by curry on 18/3/22 + */ public class TPTPGDynamicDataGenerator extends DynamicWorkloadGenerator { private static final Logger LOG = LoggerFactory.getLogger(TPTPGDynamicDataGenerator.class); private int State_Access_Skewness; // ratio of state access, following zipf distribution diff --git a/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java b/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java index e35c4bbee..6df68db0d 100644 --- a/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java @@ -77,7 +77,7 @@ public void start_evaluation(Context context, long mark_ID, int num_events) { EXPLORE(context); // MeasureTools.END_SCHEDULE_EXPLORE_TIME_MEASURE(threadId); // MeasureTools.BEGIN_SCHEDULE_USEFUL_TIME_MEASURE(threadId); - PROCESS(context, mark_ID); + PROCESS(context, mark_ID); // MeasureTools.END_SCHEDULE_USEFUL_TIME_MEASURE(threadId); // MeasureTools.END_SCHEDULE_EXPLORE_TIME_MEASURE(threadId); } while (!FINISHED(context)); From 4fbc04c47cf0d266c0c7bfabe48aa908e3cffb2b Mon Sep 17 00:00:00 2001 From: curryzjj Date: Tue, 29 Mar 2022 15:37:29 +0800 Subject: [PATCH 13/15] Workload configuration per phase --- .../datagenerator/DataGenerator.java | 5 ++ .../GSTPGDynamicDataGenerator.java | 83 +++++++++++++++--- .../apps/OB/OBTPGDynamicDataGenerator.java | 80 ++++++++++++++--- .../SLTPGDynamicDataGenerator.java | 86 +++++++++++++++---- .../apps/TP/TPTPGDynamicDataGenerator.java | 7 +- .../DynamicDataGeneratorConfig.java | 74 +++++++++++++++- .../DynamicWorkloadGenerator.java | 26 ++++-- .../initializer/GSInitializer.java | 15 ++++ .../initializer/OBInitializer.java | 16 +++- .../initializer/SLInitializer.java | 14 +++ .../initializer/TPInitializer.java | 14 +++ common/src/main/java/common/Runner.java | 57 ++++++++---- .../java/scheduler/collector/Collector.java | 50 +++++------ .../struct/op/TaskPrecedenceGraph.java | 2 +- .../java/transaction/TableInitilizer.java | 2 + .../impl/ordered/TxnManagerTStream.java | 2 +- 16 files changed, 431 insertions(+), 102 deletions(-) diff --git a/application/src/main/java/benchmark/datagenerator/DataGenerator.java b/application/src/main/java/benchmark/datagenerator/DataGenerator.java index 76590e86b..27aa86fd1 100644 --- a/application/src/main/java/benchmark/datagenerator/DataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/DataGenerator.java @@ -6,6 +6,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + /** * Data generator for benchmarks, this class contains all common methods and attributes that can be used in each application */ @@ -45,5 +47,8 @@ public void clearDataStructures() { } /* Switch configuration, used in the dynamic data generator*/ + public List getTranToDecisionConf() { + return null; + } } diff --git a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java index 91c08cbfa..e0158fd86 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java @@ -27,6 +27,7 @@ public class GSTPGDynamicDataGenerator extends DynamicWorkloadGenerator { private int Ratio_of_Overlapped_Keys; // ratio of overlapped keys in transactions, which affects the dependencies and circulars. private int Transaction_Length; private int tthread; + private int nKeyState; // control the number of txns overlap with each other. private ArrayList generatedKeys = new ArrayList<>(); // independent transactions. @@ -49,21 +50,75 @@ public GSTPGDynamicDataGenerator(DynamicDataGeneratorConfig dynamicDataConfig) { } @Override - public void switchConfiguration() { - State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; - NUM_ACCESS = dynamicDataConfig.NUM_ACCESS; - Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; - Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; - Transaction_Length = dynamicDataConfig.Transaction_Length; - - int nKeyState = dynamicDataConfig.getnKeyStates(); - int MAX_LEVEL = 256; - for (int i = 0; i < nKeyState; i++) { - idToLevel.put(i, i% MAX_LEVEL); + public void tranToDecisionConf() { + //TD,LD,PD,VDD,R_of_A,isCD,isCC, + StringBuilder stringBuilder = new StringBuilder(); + //TODO:hard code, function not sure + double td = Transaction_Length * dynamicDataConfig.getCheckpoint_interval(); + td = td *((double) Ratio_of_Overlapped_Keys/100); + stringBuilder.append(td); + stringBuilder.append(","); + double ld = Transaction_Length * dynamicDataConfig.getCheckpoint_interval(); + stringBuilder.append(ld); + stringBuilder.append(","); + double pd = Transaction_Length * dynamicDataConfig.getCheckpoint_interval() * ((double) Ratio_of_Overlapped_Keys/100) * NUM_ACCESS; + stringBuilder.append(pd); + stringBuilder.append(","); + stringBuilder.append((double) State_Access_Skewness/100); + stringBuilder.append(","); + stringBuilder.append((double) Ratio_of_Transaction_Aborts/10000); + stringBuilder.append(","); + if (AppConfig.isCyclic) { + stringBuilder.append("1,"); + } else { + stringBuilder.append("0,"); + } + if (AppConfig.complexity < 40000){ + stringBuilder.append("0,"); + } else { + stringBuilder.append("1,"); + } + stringBuilder.append(eventID+dynamicDataConfig.getShiftRate()*dynamicDataConfig.getCheckpoint_interval()*dynamicDataConfig.getTotalThreads()); + this.tranToDecisionConf.add(stringBuilder.toString()); + } + + @Override + public void switchConfiguration(String type) { + switch (type) { + case "default" : + State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; + NUM_ACCESS = dynamicDataConfig.NUM_ACCESS; + Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; + Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; + Transaction_Length = dynamicDataConfig.Transaction_Length; + + nKeyState = dynamicDataConfig.getnKeyStates(); + int MAX_LEVEL = 256; + for (int i = 0; i < nKeyState; i++) { + idToLevel.put(i, i% MAX_LEVEL); + } + keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); + configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); + p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); + break; + case "LD" : + Transaction_Length = dynamicDataConfig.Transaction_Length; + break; + case "isCyclic" : + Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; + State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; + keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); + configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); + p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); + AppConfig.isCyclic = true; + break; + case "complexity" : + AppConfig.complexity = 80000; + break; + default: + throw new IllegalStateException("Unexpected value: " + type); } - keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); - configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); - p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); + tranToDecisionConf(); } @Override diff --git a/application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java index e510cfc7e..47930bd5a 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java @@ -33,6 +33,7 @@ public class OBTPGDynamicDataGenerator extends DynamicWorkloadGenerator { private int Transaction_Length; private int tthread; private int Ratio_of_Buying; + private int nKeyState; // control the number of txns overlap with each other. private ArrayList generatedKeys = new ArrayList<>(); // independent transactions. @@ -67,24 +68,75 @@ protected void generateTuple() { } events.add(event); } + @Override + public void tranToDecisionConf() { + //TD,LD,PD,VDD,R_of_A,isCD,isCC, + StringBuilder stringBuilder = new StringBuilder(); + //TODO:hard code, function not sure + double td = Transaction_Length * dynamicDataConfig.getCheckpoint_interval() * (1 - (double)Ratio_of_Buying/100) + + ((double)Ratio_of_Buying/100) * 1 * dynamicDataConfig.getCheckpoint_interval(); + td = td *((double) Ratio_of_Overlapped_Keys/100); + stringBuilder.append(td); + stringBuilder.append(","); + double ld = Transaction_Length * dynamicDataConfig.getCheckpoint_interval() * (1 - (double)Ratio_of_Buying/100) + + ((double)Ratio_of_Buying/100) * 1 * dynamicDataConfig.getCheckpoint_interval(); + stringBuilder.append(ld); + stringBuilder.append(","); + double pd = 0; + stringBuilder.append(pd); + stringBuilder.append(","); + stringBuilder.append((double) State_Access_Skewness/100); + stringBuilder.append(","); + stringBuilder.append((double) Ratio_of_Transaction_Aborts/10000); + stringBuilder.append(","); + if (AppConfig.isCyclic) { + stringBuilder.append("1,"); + } else { + stringBuilder.append("0,"); + } + if (AppConfig.complexity < 40000){ + stringBuilder.append("0,"); + } else { + stringBuilder.append("1,"); + } + stringBuilder.append(eventID+dynamicDataConfig.getShiftRate()*dynamicDataConfig.getCheckpoint_interval()*dynamicDataConfig.getTotalThreads()); + this.tranToDecisionConf.add(stringBuilder.toString()); + } @Override - public void switchConfiguration() { - State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; - NUM_ACCESS = dynamicDataConfig.NUM_ACCESS; - Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; - Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; - Transaction_Length = dynamicDataConfig.Transaction_Length; - Ratio_of_Buying=dynamicDataConfig.Ratio_Of_Buying; + public void switchConfiguration(String type) { + switch (type){ + case "default" : + State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; + NUM_ACCESS = 1; + Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; + Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; + Transaction_Length = dynamicDataConfig.Transaction_Length; + Ratio_of_Buying=dynamicDataConfig.Ratio_Of_Buying; - int nKeyState = dynamicDataConfig.getnKeyStates(); - int MAX_LEVEL = 256; - for (int i = 0; i < nKeyState; i++) { - idToLevel.put(i, i% MAX_LEVEL); + nKeyState = dynamicDataConfig.getnKeyStates(); + int MAX_LEVEL = 256; + for (int i = 0; i < nKeyState; i++) { + idToLevel.put(i, i% MAX_LEVEL); + } + keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); + configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); + p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); + break; + case "skew" : + State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; + nKeyState = dynamicDataConfig.getnKeyStates(); + keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); + configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); + p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); + break; + case "abort" : + Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; + break; + default: + throw new IllegalStateException("Unexpected value: " + type); } - keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); - configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); - p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); + tranToDecisionConf(); } @Override diff --git a/application/src/main/java/benchmark/datagenerator/apps/SL/TPGTxnGenerator/SLTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/SL/TPGTxnGenerator/SLTPGDynamicDataGenerator.java index 34853e996..1f90ab9f1 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/SL/TPGTxnGenerator/SLTPGDynamicDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/SL/TPGTxnGenerator/SLTPGDynamicDataGenerator.java @@ -28,6 +28,7 @@ public class SLTPGDynamicDataGenerator extends DynamicWorkloadGenerator { private int Transaction_Length; // transaction length, 4 or 8 or longer private int Ratio_of_Transaction_Aborts; // ratio of transaction aborts, fail the transaction or not. i.e. transfer amount might be invalid. private int Ratio_of_Overlapped_Keys; // ratio of overlapped keys in transactions, which affects the dependencies and circulars. + private int nKeyState; // control the number of txns overlap with each other. private ArrayList generatedAcc = new ArrayList<>(); private ArrayList generatedAst = new ArrayList<>(); @@ -52,28 +53,79 @@ public SLTPGDynamicDataGenerator(DynamicDataGeneratorConfig dynamicDataConfig){ super(dynamicDataConfig); events = new ArrayList<>(nTuples); } + @Override - public void switchConfiguration() { - Ratio_Of_Deposit = dynamicDataConfig.Ratio_Of_Deposit;//0-100 (%) - State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; - Transaction_Length = 4; - Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; - Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; + public void tranToDecisionConf() { + //TD,LD,PD,VDD,R_of_A,isCD,isCC, + StringBuilder stringBuilder = new StringBuilder(); + //TODO:hard code, function not sure + double td = Transaction_Length * dynamicDataConfig.getCheckpoint_interval() * (1 - (double)Ratio_Of_Deposit/100) + + ((double)Ratio_Of_Deposit/100) * 2 * dynamicDataConfig.getCheckpoint_interval(); + td = td *((double) Ratio_of_Overlapped_Keys/100); + stringBuilder.append(td); + stringBuilder.append(","); + double ld = Transaction_Length * dynamicDataConfig.getCheckpoint_interval() * (1 - (double)Ratio_Of_Deposit/100) + + ((double)Ratio_Of_Deposit/100) * 2 * dynamicDataConfig.getCheckpoint_interval(); + stringBuilder.append(ld); + stringBuilder.append(","); + double pd = (1 - (double)Ratio_Of_Deposit/100) * Transaction_Length * dynamicDataConfig.getCheckpoint_interval() * 2 * ((double) Ratio_of_Overlapped_Keys/100); + stringBuilder.append(pd); + stringBuilder.append(","); + stringBuilder.append((double) State_Access_Skewness/100); + stringBuilder.append(","); + stringBuilder.append(Ratio_of_Transaction_Aborts/10000); + stringBuilder.append(","); + if (AppConfig.isCyclic) { + stringBuilder.append("1,"); + } else { + stringBuilder.append("0,"); + } + if (AppConfig.complexity < 40000){ + stringBuilder.append("0,"); + } else { + stringBuilder.append("1,"); + } + stringBuilder.append(eventID+dynamicDataConfig.getShiftRate()*dynamicDataConfig.getCheckpoint_interval()*dynamicDataConfig.getTotalThreads()); + this.tranToDecisionConf.add(stringBuilder.toString()); + } - int nKeyState = dynamicDataConfig.getnKeyStates(); + @Override + public void switchConfiguration(String type) { + switch (type) { + case "default" : + Ratio_Of_Deposit = dynamicDataConfig.Ratio_Of_Deposit;//0-100 (%) + State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; + Transaction_Length = 4; + Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; + Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; - // allocate levels for each key, to prevent circular. + nKeyState = dynamicDataConfig.getnKeyStates(); + // allocate levels for each key, to prevent circular. // int MAX_LEVEL = (nKeyState / dataConfig.getTotalThreads()) / 2; - int MAX_LEVEL = 256; - for (int i = 0; i < nKeyState; i++) { - idToLevel.put(i, random.nextInt(MAX_LEVEL)); + int MAX_LEVEL = 256; + for (int i = 0; i < nKeyState; i++) { + idToLevel.put(i, random.nextInt(MAX_LEVEL)); + } + // zipf state access generator + accountZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); + assetZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 123456789); + configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); + p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); + break; + case "skew" : + State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; + accountZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); + assetZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 123456789); + configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); + p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); + break; + case "PD" : + Ratio_Of_Deposit = dynamicDataConfig.Ratio_Of_Deposit;//0-100 (%) + break; + default: + throw new IllegalStateException("Unexpected value: " + type); } - // zipf state access generator - accountZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678); - assetZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 123456789); - configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState); - p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0); - + tranToDecisionConf(); } public void configure_store(double scale_factor, double theta, int tthread, int numItems) { floor_interval = (int) Math.floor(numItems / (double) tthread);//NUM_ITEMS / tthread; diff --git a/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java index 73b62df1b..7c65795cf 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java @@ -47,7 +47,12 @@ public TPTPGDynamicDataGenerator(DynamicDataGeneratorConfig dynamicDataConfig) { } @Override - public void switchConfiguration() { + public void tranToDecisionConf() { + + } + + @Override + public void switchConfiguration(String type) { State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; diff --git a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java index 6f0d955ad..d8eac4466 100644 --- a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java +++ b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java @@ -2,6 +2,7 @@ import benchmark.datagenerator.DataGeneratorConfig; import common.collections.Configuration; +import utils.AppConfig; /** * Generate dynamic workload configuration @@ -15,6 +16,8 @@ public class DynamicDataGeneratorConfig extends DataGeneratorConfig { private String app; /* Control the rate of the workload*/ private int shiftRate; + /* The workload is in which phase */ + private int phase; private int checkpoint_interval; public int State_Access_Skewness; public int NUM_ACCESS; @@ -37,6 +40,7 @@ public void initialize(Configuration config) { Transaction_Length = config.getInt("Transaction_Length", 1); Ratio_Of_Deposit = config.getInt("Ratio_Of_Deposit", 0); Ratio_Of_Buying=config.getInt("Ratio_Of_Buying",0); + phase = 0; } public int getCheckpoint_interval() { @@ -50,9 +54,17 @@ public int getShiftRate() { * Generate the configuration based on type and application * @return */ - public boolean nextDataGeneratorConfig() { - //TODO: how to switch - return false; + public String nextDataGeneratorConfig() { + switch (type){ + case "0" : + return setConfigurationForSL(); + case "1" : + return setConfigurationForOB(); + case "2" : + return setConfigurationForGS(); + default: + throw new IllegalStateException("Unexpected value: " + type); + } } public String getType() { @@ -62,4 +74,60 @@ public String getType() { public String getApp() { return app; } + + private String setConfigurationForSL() { + switch (phase){ + case 0: + phase++; + return "default"; + case 1: + phase++; + this.State_Access_Skewness = 85; + return "skew"; + case 2: + phase++; + this.Ratio_Of_Deposit = 80; + return "PD"; + default: + return null; + } + } + private String setConfigurationForOB() { + switch (phase){ + case 0: + phase++; + return "default"; + case 1: + phase++; + this.State_Access_Skewness = 85;// 100 + return "skew"; + case 2: + phase++; + this.Ratio_of_Transaction_Aborts = 8000;// 10000 + return "abort"; + default: + return null; + } + } + private String setConfigurationForGS() { + switch (phase){ + case 0: + phase++; + return "default"; + case 1: + phase++; + this.Transaction_Length = 8; + return "LD"; + case 2: + phase++; + this.Ratio_of_Transaction_Aborts = 8000;// 10000 + this.State_Access_Skewness = 85; + return "isCyclic"; + case 3: + phase++; + return "complexity"; + default: + return null; + } + } } diff --git a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java index eb68bf7d8..16cdf37fa 100644 --- a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java +++ b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java @@ -3,6 +3,9 @@ import benchmark.datagenerator.DataGenerator; import benchmark.datagenerator.DataGeneratorConfig; +import java.util.ArrayList; +import java.util.List; + /** * Generate dynamic workload @@ -12,33 +15,40 @@ * Created by curry on 16/3/22. */ public abstract class DynamicWorkloadGenerator extends DataGenerator { + protected List tranToDecisionConf =new ArrayList<>(); protected DynamicDataGeneratorConfig dynamicDataConfig; public DynamicWorkloadGenerator(DynamicDataGeneratorConfig dynamicDataConfig) { super(dynamicDataConfig); this.dynamicDataConfig=dynamicDataConfig; } - private String[] tranToDecisionConf() { - return null; - } + /** + * Each application may be different + */ + public abstract void tranToDecisionConf(); @Override public void generateStream() { //Init the Configuration - switchConfiguration(); for (int tupleNumber = 0; tupleNumber < nTuples + dynamicDataConfig.getTotalThreads(); tupleNumber++) { - if (tupleNumber%dynamicDataConfig.getCheckpoint_interval()* dynamicDataConfig.getShiftRate()* dynamicDataConfig.getTotalThreads() == 0) { - if (dynamicDataConfig.nextDataGeneratorConfig()) { - switchConfiguration(); + if (tupleNumber%(dynamicDataConfig.getCheckpoint_interval()* dynamicDataConfig.getShiftRate()* dynamicDataConfig.getTotalThreads()) == 0) { + String type = dynamicDataConfig.nextDataGeneratorConfig(); + if (type != null) { + switchConfiguration(type); } } generateTuple(); } } - public abstract void switchConfiguration(); + public abstract void switchConfiguration(String type); @Override public DynamicDataGeneratorConfig getDataConfig() { return dynamicDataConfig; } + + @Override + public List getTranToDecisionConf() { + return tranToDecisionConf; + } } diff --git a/application/src/main/java/common/topology/transactional/initializer/GSInitializer.java b/application/src/main/java/common/topology/transactional/initializer/GSInitializer.java index 880fbd6cf..b00331514 100644 --- a/application/src/main/java/common/topology/transactional/initializer/GSInitializer.java +++ b/application/src/main/java/common/topology/transactional/initializer/GSInitializer.java @@ -8,6 +8,7 @@ import benchmark.datagenerator.apps.GS.TPGTxnGenerator.GSTPGDynamicDataGenerator; import benchmark.datagenerator.apps.SL.TPGTxnGenerator.SLTPGDataGeneratorConfig; import benchmark.dynamicWorkloadGenerator.DynamicDataGeneratorConfig; +import benchmark.dynamicWorkloadGenerator.DynamicWorkloadGenerator; import common.collections.Configuration; import common.collections.OsUtils; import common.param.TxnEvent; @@ -382,6 +383,11 @@ public void store(String file_name) throws IOException { w.close(); } + @Override + public List getTranToDecisionConf() { + return dataGenerator.getTranToDecisionConf(); + } + private RecordSchema MicroTableSchema() { List dataBoxes = new ArrayList<>(); List fieldNames = new ArrayList<>(); @@ -397,6 +403,15 @@ public void creates_Table(Configuration config) { db.createTable(s, "MicroTable"); try { prepare_input_events(config.getInt("totalEvents")); + if (getTranToDecisionConf() != null && getTranToDecisionConf().size() !=0){ + StringBuilder stringBuilder = new StringBuilder(); + for(String decision:getTranToDecisionConf()){ + stringBuilder.append(decision); + stringBuilder.append(";"); + } + stringBuilder.deleteCharAt(stringBuilder.length()-1); + config.put("WorkloadConfig",stringBuilder.toString()); + } } catch (IOException e) { e.printStackTrace(); } diff --git a/application/src/main/java/common/topology/transactional/initializer/OBInitializer.java b/application/src/main/java/common/topology/transactional/initializer/OBInitializer.java index 00a0d30b8..30943bca6 100644 --- a/application/src/main/java/common/topology/transactional/initializer/OBInitializer.java +++ b/application/src/main/java/common/topology/transactional/initializer/OBInitializer.java @@ -227,7 +227,7 @@ private void loadOnlineBiddingEvents(BufferedReader reader, int totalEvents, boo String[] split = txn.split(","); if (split.length > 3) { int npid = (int) (Long.parseLong(split[1]) / partitionOffset); - int keyLength = NUM_ACCESS*Transaction_Length; + int keyLength = 1*Transaction_Length; HashMap pids = new HashMap<>(); int[] keys = new int[keyLength]; for (int i = 1; i < keyLength+1; i++) { @@ -338,11 +338,25 @@ public void store(String file_name) throws IOException { w.close(); } + @Override + public List getTranToDecisionConf() { + return dataGenerator.getTranToDecisionConf(); + } + public void creates_Table(Configuration config) { RecordSchema s = Goods(); db.createTable(s, "goods"); try { prepare_input_events(config.getInt("totalEvents")); + if (getTranToDecisionConf() != null && getTranToDecisionConf().size() != 0){//input data already exist + StringBuilder stringBuilder = new StringBuilder(); + for(String decision:getTranToDecisionConf()){ + stringBuilder.append(decision); + stringBuilder.append(";"); + } + stringBuilder.deleteCharAt(stringBuilder.length()-1); + config.put("WorkloadConfig",stringBuilder.toString()); + } } catch (IOException e) { e.printStackTrace(); } diff --git a/application/src/main/java/common/topology/transactional/initializer/SLInitializer.java b/application/src/main/java/common/topology/transactional/initializer/SLInitializer.java index 992d1aa54..e216336f5 100644 --- a/application/src/main/java/common/topology/transactional/initializer/SLInitializer.java +++ b/application/src/main/java/common/topology/transactional/initializer/SLInitializer.java @@ -325,6 +325,11 @@ public void store(String file_name) throws IOException { } + @Override + public List getTranToDecisionConf() { + return dataGenerator.getTranToDecisionConf(); + } + private void loadTransferDepositEvents(BufferedReader reader, int totalEvents, boolean shufflingActive, int[] p_bids) throws IOException { String txn = reader.readLine(); int count = 0; @@ -417,6 +422,15 @@ public void creates_Table(Configuration config) { db.createTable(b, "bookEntries"); try { prepare_input_events(config.getInt("totalEvents")); + if (getTranToDecisionConf() != null && getTranToDecisionConf().size() != 0){ + StringBuilder stringBuilder = new StringBuilder(); + for(String decision:getTranToDecisionConf()){ + stringBuilder.append(decision); + stringBuilder.append(";"); + } + stringBuilder.deleteCharAt(stringBuilder.length()-1); + config.put("WorkloadConfig",stringBuilder.toString()); + } } catch (IOException e) { e.printStackTrace(); } diff --git a/application/src/main/java/common/topology/transactional/initializer/TPInitializer.java b/application/src/main/java/common/topology/transactional/initializer/TPInitializer.java index 8d6f06072..0f1aca6a3 100644 --- a/application/src/main/java/common/topology/transactional/initializer/TPInitializer.java +++ b/application/src/main/java/common/topology/transactional/initializer/TPInitializer.java @@ -294,6 +294,11 @@ private void shuffleEvents(ArrayList txnEvents, int totalEvents) { public void store(String file_name) { } + @Override + public List getTranToDecisionConf() { + return dataGenerator.getTranToDecisionConf(); + } + protected String getConfigKey(String template) { return String.format(template, "tptxn");//TODO: make it flexible in future. } @@ -305,6 +310,15 @@ public void creates_Table(Configuration config) { db.createTable(b, "segment_cnt"); try { prepare_input_events(config.getInt("totalEvents")); + if (getTranToDecisionConf() != null && getTranToDecisionConf().size() != 0){ + StringBuilder stringBuilder = new StringBuilder(); + for(String decision:getTranToDecisionConf()){ + stringBuilder.append(decision); + stringBuilder.append(";"); + } + stringBuilder.deleteCharAt(stringBuilder.length()-1); + config.put("WorkloadConfig",stringBuilder.toString()); + } } catch (IOException e) { e.printStackTrace(); } diff --git a/common/src/main/java/common/Runner.java b/common/src/main/java/common/Runner.java index 5bd3a98dc..3e779f201 100644 --- a/common/src/main/java/common/Runner.java +++ b/common/src/main/java/common/Runner.java @@ -19,10 +19,10 @@ public abstract class Runner implements IRunner { * Workload Specific Parameters. */ @Parameter(names = {"-a", "--app"}, description = "The application to be executed") - //public String application = "StreamLedger"; + public String application = "StreamLedger"; //public String application = "GrepSum"; //public String application = "OnlineBiding"; - public String application = "TollProcessing"; + //public String application = "TollProcessing"; @Parameter(names = {"-t", "--topology-name"}, required = false, description = "The name of the application") public String topologyName; @Parameter(names = {"--COMPUTE_COMPLEXITY"}, description = "COMPUTE_COMPLEXITY per event") @@ -33,7 +33,7 @@ public abstract class Runner implements IRunner { public int NUM_ITEMS = 100_000;// // public int NUM_ITEMS = 500;// @Parameter(names = {"--NUM_ACCESS"}, description = "Number of state access per transaction") - public int NUM_ACCESS = 10;// + public int NUM_ACCESS = 5;// @Parameter(names = {"--ratio_of_read"}, description = "ratio_of_read") public double ratio_of_read = 0.0; //<=1 @Parameter(names = {"--ratio_of_multi_partition"}, description = "ratio_of_multi_partition") @@ -67,8 +67,8 @@ public abstract class Runner implements IRunner { @Parameter(names = {"--partition"}, description = "Partitioning database. It must be enabled for S-Store scheme and it is optional for TStream scheme.") public boolean enable_partition = false; @Parameter(names = {"--scheduler"}, description = "Scheduler for TStream.") - public String scheduler = "OP_BFS"; - // public String scheduler = "OG_BFS_A"; + public String scheduler = "OP_BFS_A"; + //public String scheduler = "OG_BFS_A"; // public String scheduler = "OG_DFS"; // public String scheduler = "OG_DFS_A"; // public String scheduler = "OG_NS"; @@ -91,13 +91,13 @@ public abstract class Runner implements IRunner { @Parameter(names = {"--isRuntime"}, description = "Collect runtime information") public boolean isRuntime = false; @Parameter(names = {"--isDynamic"}, description = "Dynamic Workload") - public boolean isDynamic = false; + public boolean isDynamic = true; @Parameter(names = {"--schedulerPool"}, description = "Schedulers in the SchedulerPool[OG_DFS,OP_DFS]") - public String schedulerPools = "OP_NS_A,OP_BFS_A,OP_BFS,OG_BFS"; + public String schedulerPools = ""; @Parameter(names = {"--defaultScheduler"}, description = "Default scheduler") - public String defaultScheduler = "OG_BFS"; + public String defaultScheduler = ""; @Parameter(names = {"--bottomLine"}, description = "BottomLine for(TD,LD,PD,SUM,VDD,R_of_A)") - public String bottomLine = "2,3,5,10,0.6,0.7"; + public String bottomLine = ""; @Parameter(names = {"--WorkloadConfig"}, description = "WorkloadConfigs(TD,LD,PD,VDD,R_of_A,isCD,isCC,markId)") public String WorkloadConfig = "1,3,4,0.5,0.6,1,1,39999;1,6,4,0.5,0.7,1,1,79999;3,6,3,0.5,0.7,1,1,100000"; @@ -105,10 +105,9 @@ public abstract class Runner implements IRunner { * Dynamic workload */ @Parameter(names = {"--workloadType"}, description = "which type of dynamic workload") - public String workloadType = ""; @Parameter(names = {"-shiftRate-"}, description = "control the rate of the workload shift") - public int shiftRate = 5; + public int shiftRate = 2; /** * Benchmarking Specific Parameters. @@ -132,12 +131,12 @@ public abstract class Runner implements IRunner { * generator parameters */ @Parameter(names = {"--checkpoint_interval"}, description = "checkpoint interval (#tuples)") - public int checkpoint_interval = 2500;//checkpoint per thread. + public int checkpoint_interval = 1500;//checkpoint per thread. @Parameter(names = {"--generator"}, description = "Generator for TStream.") public String generator = "TPGGenerator"; // public String generator = "OCGenerator"; @Parameter(names = {"--totalEvents"}, description = "Total number of events to process.") - public int totalEvents = 10000; + public int totalEvents = 48000; @Parameter(names = {"--deposit_ratio"}, description = "Ratio of deposit for SL.") public Integer Ratio_Of_Deposit = 25; @@ -152,7 +151,7 @@ public abstract class Runner implements IRunner { public Integer Ratio_of_Overlapped_Keys = 10; @Parameter(names = {"--abort_ratio"}, description = "Ratio of transaction aborts.") - public Integer Ratio_of_Transaction_Aborts = 0; + public Integer Ratio_of_Transaction_Aborts = 500; @Parameter(names = {"--txn_length"}, description = "Transaction Length.") public Integer Transaction_Length = 1; @@ -164,7 +163,7 @@ public abstract class Runner implements IRunner { public int isCyclic = 0; @Parameter(names = {"--complexity"}, description = "Dummy UDF complexity for state access process.") - public Integer complexity = 100000; + public Integer complexity = 10000; public Runner() { CFG_PATH = "/config/%s.properties"; @@ -188,7 +187,6 @@ public void initializeCfg(HashMap config) { config.put("machine", machine); config.put("totalEvents", totalEvents); config.put("rootFilePath", rootPath); - config.put("scheduler", scheduler); config.put("generator", generator); config.put("fanoutDist", fanoutDist); config.put("idGenType", idGenType); @@ -240,16 +238,39 @@ public void initializeCfg(HashMap config) { config.put("size_tuple", size_tuple); config.put("verbose", verbose); + config.put("application",application); + switch(application) { + case "StreamLedger" : + workloadType = "0"; + bottomLine = "300,3000,500,4000,0.2,0.2";//TD,LD,PD,SUM,VDD,R_of_A + schedulerPools = "OP_BFS_A,OP_NS_A,OG_NS_A"; + defaultScheduler = "OP_BFS_A"; + break; + case "OnlineBiding" : + workloadType ="1"; + bottomLine = "500,5000,1,6000,0.2,0.2";//TD,LD,PD,SUM,VDD,R_of_A + schedulerPools = "OG_BFS_A,OG_NS_A,OG_NS"; + defaultScheduler = "OG_BFS_A"; + break; + case "GrepSum" : + workloadType ="2"; + bottomLine = "500,5000,6500,3000,0.2,0.2";//TD,LD,PD,SUM,VDD,R_of_A + schedulerPools = "OP_NS_A,OG_BFS_A,OP_NS,OP_NS_A"; + defaultScheduler = "OP_NS_A"; + break; + default: + throw new IllegalStateException("Unexpected value: " + application); + } /* Dynamic switch scheduler*/ config.put("isDynamic",isDynamic); config.put("schedulersPool",schedulerPools); - config.put("defaultScheduler",scheduler); + config.put("defaultScheduler",defaultScheduler); + config.put("scheduler", defaultScheduler); config.put("isRuntime",isRuntime); config.put("bottomLine",bottomLine); config.put("WorkloadConfig",WorkloadConfig); /* Dynamic Workload Configuration*/ - config.put("application",application); config.put("workloadType",workloadType); config.put("shiftRate",shiftRate); diff --git a/state-engine/src/main/java/scheduler/collector/Collector.java b/state-engine/src/main/java/scheduler/collector/Collector.java index 6c9a400c7..0c15963bf 100644 --- a/state-engine/src/main/java/scheduler/collector/Collector.java +++ b/state-engine/src/main/java/scheduler/collector/Collector.java @@ -12,10 +12,10 @@ public class Collector { Num of TD,LD,PD */ private HashMap Num_of_dependency; - private int B_TD; - private int B_LD; - private int B_PD; - private int B_SUM_D; + private double B_TD; + private double B_LD; + private double B_PD; + private double B_SUM_D; /* Vertex Degree Distribution */ @@ -37,7 +37,7 @@ public class Collector { private boolean isRuntime = true; private int threadCount; private final List workloadConfig=new ArrayList<>(); - private final HashMap currentWorkload=new HashMap<>(); + private final HashMap currentWorkload=new HashMap<>();// /** * Init the collector */ @@ -52,12 +52,12 @@ public void InitCollector(int threadCount){ */ public void setBottomLine(String bottomLine){ String b_ls[]=bottomLine.split(","); - this.B_TD= Integer.parseInt(b_ls[0]); - this.B_LD= Integer.parseInt(b_ls[1]); - this.B_PD= Integer.parseInt(b_ls[2]); - this.B_SUM_D=Integer.parseInt(b_ls[3]); - this.B_VDD= Double.parseDouble(b_ls[4]); - this.B_R_of_A= Double.parseDouble(b_ls[5]); + this.B_TD = Double.parseDouble(b_ls[0]); + this.B_LD = Double.parseDouble(b_ls[1]); + this.B_PD = Double.parseDouble(b_ls[2]); + this.B_SUM_D = Double.parseDouble(b_ls[3]); + this.B_VDD = Double.parseDouble(b_ls[4]); + this.B_R_of_A = Double.parseDouble(b_ls[5]); } /** * Load the workload config if not collecting information runtime @@ -79,7 +79,7 @@ public void setWorkloadConfig(String config){ * 0000 0001 0010 0011 0100 0101 0110 0111 1000 1001 1010 1011 * @return */ - private String decisionTree(int TD,int LD,int PD,double VDD,boolean isCyclicDependency,boolean isComputationComplexity,double R_of_A){ + private String decisionTree(double TD,double LD,double PD,double VDD,boolean isCyclicDependency,boolean isComputationComplexity,double R_of_A){ String schedulers[]={"OG_NS","OG_NS_A","OP_NS","OP_NS_A","OG_BFS","OG_BFS_A","OP_BFS","OP_BFS_A","OG_DFS","OG_DFS_A","OP_DFS","OP_DFS_A"}; int flag=0; if((TD+PD+LD)>B_SUM_D){ @@ -100,11 +100,11 @@ private String decisionTree(int TD,int LD,int PD,double VDD,boolean isCyclicDepe private String getDecisionFromConf(int threadId){ int workloadId=currentWorkload.get(threadId); String para[]=workloadConfig.get(workloadId); - int TD= Integer.parseInt(para[0]); - int LD= Integer.parseInt(para[1]); - int PD= Integer.parseInt(para[2]); - double VDD=Double.parseDouble(para[3]); - double R_of_A=Double.parseDouble(para[4]); + double TD = Double.parseDouble(para[0]); + double LD = Double.parseDouble(para[1]); + double PD = Double.parseDouble(para[2]); + double VDD = Double.parseDouble(para[3]); + double R_of_A = Double.parseDouble(para[4]); boolean isCD = false,isCC=false; if(para[5].equals("1")){ isCD=true; @@ -131,13 +131,15 @@ public boolean timeToSwitch(long markId,int threadId){ //TODO:collect information runtime return false; }else{ - int workloadId=this.currentWorkload.get(threadId); - long currentId= Long.parseLong(workloadConfig.get(workloadId)[workloadConfig.get(workloadId).length-1]); - if (currentId<=markId+threadCount-threadId-1){ - currentWorkload.put(threadId,currentWorkload.get(threadId)+1); - return true; - }else - return false; + int workloadId = this.currentWorkload.get(threadId); + if (workloadId < workloadConfig.size() -1){ + long currentId = Long.parseLong(workloadConfig.get(workloadId)[workloadConfig.get(workloadId).length-1]); + if (currentId <= markId+threadCount-threadId){ + currentWorkload.put(threadId,currentWorkload.get(threadId)+1); + return true; + } + } + return false; } } } diff --git a/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java index f0d50cc8e..c413af2fa 100644 --- a/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java @@ -126,7 +126,7 @@ public void setOCs(Context context) { operationChains.get("segment_cnt").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, cntOC); ocs.add(speedOC); ocs.add(cntOC); - } else if (app ==3) { + } else if (app == 3) { OperationChain gsOC = context.createTask("goods", _key); operationChains.get("goods").threadOCsMap.get(context.thisThreadId).holder_v1.put(_key, gsOC); ocs.add(gsOC); diff --git a/state-engine/src/main/java/transaction/TableInitilizer.java b/state-engine/src/main/java/transaction/TableInitilizer.java index f9a8147f9..0dfb52c71 100644 --- a/state-engine/src/main/java/transaction/TableInitilizer.java +++ b/state-engine/src/main/java/transaction/TableInitilizer.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.List; import java.util.Set; import java.util.SplittableRandom; @@ -151,5 +152,6 @@ public void prepare_input_events(int total_events) throws IOException { protected abstract void Load() throws IOException; public abstract void store(String file_path) throws IOException; + public abstract List getTranToDecisionConf(); } diff --git a/state-engine/src/main/java/transaction/impl/ordered/TxnManagerTStream.java b/state-engine/src/main/java/transaction/impl/ordered/TxnManagerTStream.java index 499e42fac..580944cb7 100644 --- a/state-engine/src/main/java/transaction/impl/ordered/TxnManagerTStream.java +++ b/state-engine/src/main/java/transaction/impl/ordered/TxnManagerTStream.java @@ -64,7 +64,7 @@ public void start_evaluate(int thread_Id, long mark_ID, int num_events) throws I MeasureTools.SCHEDULE_TIME_RECORD(thread_Id, num_events); //Sync to switch scheduler(more overhead) decide by the mark_ID or runtime information if(enableDynamic&&collector.timeToSwitch(mark_ID,thread_Id)){ - if(thread_Id==0){ + if(thread_Id == 0){ String schedulerType=collector.getDecision(thread_Id); this.SwitchScheduler(schedulerType); if(enable_log) log.info("Current Scheduler is "+schedulerType+" MarkId "+mark_ID); From 79a9ac63172e5eb74c50b5c1a8d3f603402ed542 Mon Sep 17 00:00:00 2001 From: curryzjj Date: Thu, 31 Mar 2022 21:01:28 +0800 Subject: [PATCH 14/15] Add Groups for TP --- affinity/src/main/c/Makefile | 6 +- .../GSTPGDynamicDataGenerator.java | 2 + .../apps/OB/OBTPGDynamicDataGenerator.java | 2 + .../SLTPGDynamicDataGenerator.java | 2 + .../apps/TP/TPTPGDynamicDataGenerator.java | 10 +- .../DynamicDataGeneratorConfig.java | 34 +++- .../DynamicWorkloadGenerator.java | 1 - .../common/bolts/transactional/gs/GSBolt.java | 3 +- .../bolts/transactional/gs/GSBolt_ts.java | 3 - .../bolts/transactional/ob/OBBolt_ts_s.java | 3 +- .../bolts/transactional/tp/TPBolt_ts_s.java | 3 +- .../main/java/common/sink/MeasureSink.java | 1 - .../main/java/runners/MorphStreamRunner.java | 2 +- common/src/main/java/common/Runner.java | 51 +++++- .../main/java/common/platform/Platform.java | 12 +- .../src/main/java/profiler/MeasureTools.java | 28 +++- .../src/main/java/profiler/Metrics.java | 10 ++ .../java/scheduler/collector/Collector.java | 8 +- .../main/java/scheduler/impl/IScheduler.java | 1 + .../java/scheduler/impl/og/OGScheduler.java | 6 + .../impl/og/structured/OGSScheduler.java | 2 +- .../java/scheduler/impl/op/OPScheduler.java | 8 +- .../java/scheduler/struct/og/TableOCs.java | 4 +- .../struct/og/TaskPrecedenceGraph.java | 15 +- .../java/scheduler/struct/op/TableOCs.java | 4 +- .../struct/op/TaskPrecedenceGraph.java | 36 ++-- .../src/main/java/transaction/TxnManager.java | 29 +++- .../impl/TxnManagerDedicatedAsy.java | 157 +++++++++++++++--- .../impl/ordered/TxnManagerTStream.java | 10 +- .../src/main/java/utils/SOURCE_CONTROL.java | 16 +- .../operators/api/TransactionalBolt.java | 2 +- .../affinity/SequentialBinding.java | 2 +- .../main/java/execution/ExecutionManager.java | 2 + 33 files changed, 382 insertions(+), 93 deletions(-) diff --git a/affinity/src/main/c/Makefile b/affinity/src/main/c/Makefile index 6bb98b2a4..86e88454f 100755 --- a/affinity/src/main/c/Makefile +++ b/affinity/src/main/c/Makefile @@ -19,13 +19,13 @@ JNI_JAVA_SOURCES := $(patsubst %,$(TARGET_DIR)/%.class,$(subst .,/,$(JAVA_CLASSE JAVA_BUILD_DIR := $(TARGET_DIR) -JAVA_HOME ?= /usr/java/default -#JAVA_HOME :=/Library/Java/JavaVirtualMachines/jdk1.8.0_191.jdk/Contents/Home +#JAVA_HOME ?= /usr/java/default +JAVA_HOME :=/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home JAVA_LIB := $(JAVA_HOME)/jre/lib JVM_SHARED_LIBS := -L$(JAVA_LIB)/amd64/server -L$(JAVA_LIB)/i386/server -L$(JAVA_LIB)/amd64/jrockit/ -L$(JAVA_LIB)/i386/jrockit/ -L$(JAVA_LIB)/ppc64le/server -L$(JAVA_LIB)/ppc64le/jrockit/ CXX=g++ -INCLUDES := -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(WORKING_DIR) +INCLUDES := -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(WORKING_DIR) -I $(JAVA_HOME)/include/darwin # classpath for javah ifdef CLASSPATH diff --git a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java index e0158fd86..8444ba3bc 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java @@ -115,6 +115,8 @@ public void switchConfiguration(String type) { case "complexity" : AppConfig.complexity = 80000; break; + case "unchanging" : + break; default: throw new IllegalStateException("Unexpected value: " + type); } diff --git a/application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java index 47930bd5a..17e645aab 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java @@ -133,6 +133,8 @@ public void switchConfiguration(String type) { case "abort" : Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; break; + case "unchanging" : + break; default: throw new IllegalStateException("Unexpected value: " + type); } diff --git a/application/src/main/java/benchmark/datagenerator/apps/SL/TPGTxnGenerator/SLTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/SL/TPGTxnGenerator/SLTPGDynamicDataGenerator.java index 1f90ab9f1..f61beb1fc 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/SL/TPGTxnGenerator/SLTPGDynamicDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/SL/TPGTxnGenerator/SLTPGDynamicDataGenerator.java @@ -122,6 +122,8 @@ public void switchConfiguration(String type) { case "PD" : Ratio_Of_Deposit = dynamicDataConfig.Ratio_Of_Deposit;//0-100 (%) break; + case "unchanging" : + break; default: throw new IllegalStateException("Unexpected value: " + type); } diff --git a/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java index 7c65795cf..a24bbc1ac 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/TP/TPTPGDynamicDataGenerator.java @@ -40,6 +40,7 @@ public class TPTPGDynamicDataGenerator extends DynamicWorkloadGenerator { private HashMap nGeneratedIds = new HashMap<>(); private ArrayList events; private int eventID = 0; + private boolean enableGroup; private HashMap idToLevel = new HashMap<>(); public TPTPGDynamicDataGenerator(DynamicDataGeneratorConfig dynamicDataConfig) { super(dynamicDataConfig); @@ -56,6 +57,7 @@ public void switchConfiguration(String type) { State_Access_Skewness = dynamicDataConfig.State_Access_Skewness; Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; + this.enableGroup=dynamicDataConfig.enableGroup; int nKeyState = dynamicDataConfig.getnKeyStates(); int MAX_LEVEL = 256; for (int i = 0; i < nKeyState; i++) { @@ -107,7 +109,13 @@ private Event randomTPEvent() { int id; if (!isUnique) { if (enable_states_partition) { - int partitionId = key_to_partition(p_generator.next()); + //Must Determine partitionId from EventId + int partitionId; + if (enableGroup) { + partitionId = eventID % dynamicDataConfig.getTotalThreads(); + } else { + partitionId = key_to_partition(p_generator.next()); + } id = getKey(partitionedKeyZipf[partitionId],partitionId,generatedKeys); } else { id = getKey(keyZipf,generatedKeys); diff --git a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java index d8eac4466..28e7b2327 100644 --- a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java +++ b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java @@ -26,6 +26,7 @@ public class DynamicDataGeneratorConfig extends DataGeneratorConfig { public int Transaction_Length; public int Ratio_Of_Deposit; public int Ratio_Of_Buying; + public boolean enableGroup; public void initialize(Configuration config) { super.initialize(config); @@ -33,6 +34,7 @@ public void initialize(Configuration config) { this.type = config.getString("workloadType"); this.shiftRate = config.getInt("shiftRate"); this.checkpoint_interval = config.getInt("checkpoint"); + enableGroup = config.getBoolean("isGroup"); NUM_ACCESS = config.getInt("NUM_ACCESS", 0); State_Access_Skewness = config.getInt("State_Access_Skewness", 0); Ratio_of_Overlapped_Keys = config.getInt("Ratio_of_Overlapped_Keys", 0); @@ -62,6 +64,8 @@ public String nextDataGeneratorConfig() { return setConfigurationForOB(); case "2" : return setConfigurationForGS(); + case "3" : + return setConfigurationForTP(); default: throw new IllegalStateException("Unexpected value: " + type); } @@ -75,19 +79,27 @@ public String getApp() { return app; } + //User defined private String setConfigurationForSL() { switch (phase){ case 0: - phase++; + phase ++; return "default"; case 1: - phase++; - this.State_Access_Skewness = 85; - return "skew"; + case 4: + phase ++; + return "unchanging"; case 2: - phase++; - this.Ratio_Of_Deposit = 80; - return "PD"; + case 3: + phase ++; + this.State_Access_Skewness = this.State_Access_Skewness + 30; + return "skew"; + case 5: + case 6: + case 7: + phase ++; + this.Ratio_Of_Deposit = this.Ratio_Of_Deposit + 20 ; + return "skew" ; default: return null; } @@ -130,4 +142,12 @@ private String setConfigurationForGS() { return null; } } + private String setConfigurationForTP(){ + if (phase==0){ + phase ++; + return "default"; + } else { + return null; + } + } } diff --git a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java index 16cdf37fa..279506b80 100644 --- a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java +++ b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicWorkloadGenerator.java @@ -1,7 +1,6 @@ package benchmark.dynamicWorkloadGenerator; import benchmark.datagenerator.DataGenerator; -import benchmark.datagenerator.DataGeneratorConfig; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/common/bolts/transactional/gs/GSBolt.java b/application/src/main/java/common/bolts/transactional/gs/GSBolt.java index 3fdb0745b..b7a04ff90 100644 --- a/application/src/main/java/common/bolts/transactional/gs/GSBolt.java +++ b/application/src/main/java/common/bolts/transactional/gs/GSBolt.java @@ -37,8 +37,9 @@ protected void TXN_PROCESS(long _bid) throws DatabaseException, InterruptedExcep protected boolean READ_CORE(MicroEvent event) { for (int i = 0; i < event.TOTAL_NUM_ACCESS; ++i) { SchemaRecordRef ref = event.getRecord_refs()[i]; - if (ref.isEmpty()) + if (ref.isEmpty()) { return false;//not yet processed. + } DataBox dataBox = ref.getRecord().getValues().get(1); int read_result = Integer.parseInt(dataBox.getString().trim()); event.result[i] = read_result; diff --git a/application/src/main/java/common/bolts/transactional/gs/GSBolt_ts.java b/application/src/main/java/common/bolts/transactional/gs/GSBolt_ts.java index 66f0a7c02..d2354fd6d 100644 --- a/application/src/main/java/common/bolts/transactional/gs/GSBolt_ts.java +++ b/application/src/main/java/common/bolts/transactional/gs/GSBolt_ts.java @@ -132,9 +132,6 @@ public void execute(Tuple in) throws InterruptedException, DatabaseException, Br { MeasureTools.BEGIN_TXN_TIME_MEASURE(thread_Id); { - if (i==1){ - System.out.println(); - } transactionManager.start_evaluate(thread_Id, in.getBID(), num_events);//start lazy evaluation in transaction manager. i++; READ_REQUEST_CORE(); diff --git a/application/src/main/java/common/bolts/transactional/ob/OBBolt_ts_s.java b/application/src/main/java/common/bolts/transactional/ob/OBBolt_ts_s.java index 5b1d63b6f..e16592633 100644 --- a/application/src/main/java/common/bolts/transactional/ob/OBBolt_ts_s.java +++ b/application/src/main/java/common/bolts/transactional/ob/OBBolt_ts_s.java @@ -64,7 +64,6 @@ public void execute(Tuple in) throws InterruptedException, DatabaseException, Br BUYING_REQUEST_CORE(); } MeasureTools.END_TXN_TIME_MEASURE(thread_Id); - BUYING_REQUEST_POST(); MeasureTools.BEGIN_POST_TIME_MEASURE(thread_Id); BUYING_REQUEST_POST(); MeasureTools.END_POST_TIME_MEASURE_ACC(thread_Id); @@ -73,7 +72,7 @@ public void execute(Tuple in) throws InterruptedException, DatabaseException, Br alertEvents = 0; toppingEvents = 0; } - MeasureTools.END_TOTAL_TIME_MEASURE_TS(thread_Id,readSize); + MeasureTools.END_TOTAL_TIME_MEASURE_TS(thread_Id,num_events); }else { execute_ts_normal(in); } diff --git a/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts_s.java b/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts_s.java index 764a2db88..7f1b48ae2 100644 --- a/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts_s.java +++ b/application/src/main/java/common/bolts/transactional/tp/TPBolt_ts_s.java @@ -21,6 +21,7 @@ import static common.CONTROL.enable_latency_measurement; import static common.constants.TPConstants.Constant.NUM_SEGMENTS; +import static profiler.Metrics.NUM_ITEMS; public class TPBolt_ts_s extends TPBolt { private static final Logger LOG= LoggerFactory.getLogger(TPBolt_ts_s.class); @@ -36,7 +37,7 @@ public TPBolt_ts_s(int fid){ public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { super.initialize(thread_Id, thisTaskId, graph); transactionManager=new TxnManagerTStream(db.getStorageManager(), this.context.getThisComponentId(),thread_Id, - NUM_SEGMENTS,this.context.getThisComponent().getNumTasks(),config.getString("scheduler","BF")); + NUM_ITEMS,this.context.getThisComponent().getNumTasks(),config.getString("scheduler","BF")); LREvents = new ArrayDeque<>(); } public void loadDB(Map conf, TopologyContext context, OutputCollector collector) { diff --git a/application/src/main/java/common/sink/MeasureSink.java b/application/src/main/java/common/sink/MeasureSink.java index c2ed47ba6..792ed7c93 100644 --- a/application/src/main/java/common/sink/MeasureSink.java +++ b/application/src/main/java/common/sink/MeasureSink.java @@ -170,7 +170,6 @@ protected void latency_measure(Tuple input) { // } // } latency_map.add(System.nanoTime() - input.getLong(1)); - } } diff --git a/application/src/main/java/runners/MorphStreamRunner.java b/application/src/main/java/runners/MorphStreamRunner.java index 93acc10cc..f31d20447 100644 --- a/application/src/main/java/runners/MorphStreamRunner.java +++ b/application/src/main/java/runners/MorphStreamRunner.java @@ -305,7 +305,7 @@ public void run() throws InterruptedException { } catch (IOException e) { e.printStackTrace(); } - METRICS_REPORT(config.getInt("CCOption", 0), file, tthread, rt); + METRICS_REPORT(config.getInt("CCOption", 0), file, tthread, rt,config.getInt("phaseNum")); } }//end of profile. } diff --git a/common/src/main/java/common/Runner.java b/common/src/main/java/common/Runner.java index 3e779f201..88c24fd71 100644 --- a/common/src/main/java/common/Runner.java +++ b/common/src/main/java/common/Runner.java @@ -91,7 +91,7 @@ public abstract class Runner implements IRunner { @Parameter(names = {"--isRuntime"}, description = "Collect runtime information") public boolean isRuntime = false; @Parameter(names = {"--isDynamic"}, description = "Dynamic Workload") - public boolean isDynamic = true; + public int isDynamic = 0; @Parameter(names = {"--schedulerPool"}, description = "Schedulers in the SchedulerPool[OG_DFS,OP_DFS]") public String schedulerPools = ""; @Parameter(names = {"--defaultScheduler"}, description = "Default scheduler") @@ -106,8 +106,20 @@ public abstract class Runner implements IRunner { */ @Parameter(names = {"--workloadType"}, description = "which type of dynamic workload") public String workloadType = ""; - @Parameter(names = {"-shiftRate-"}, description = "control the rate of the workload shift") - public int shiftRate = 2; + @Parameter(names = {"--shiftRate"}, description = "control the rate of the workload shift") + public int shiftRate = 1; + @Parameter(names = {"--phaseNum"}, description = "How many phases") + public int phaseNum = 2; + + /** + * Scheduler for group + */ + @Parameter(names = {"--isGroup"}, description = "Group for TP") + public int isGroup = 0; + @Parameter(names = {"--groupNum"}, description = "How many groups") + public int groupNum = 2; + @Parameter(names = {"--SchedulersForGroup"}, description = "Schedulers [OG_DFS,OP_DFS]") + public String SchedulersForGroup = ""; /** * Benchmarking Specific Parameters. @@ -131,12 +143,12 @@ public abstract class Runner implements IRunner { * generator parameters */ @Parameter(names = {"--checkpoint_interval"}, description = "checkpoint interval (#tuples)") - public int checkpoint_interval = 1500;//checkpoint per thread. + public int checkpoint_interval = 2500;//checkpoint per thread. @Parameter(names = {"--generator"}, description = "Generator for TStream.") public String generator = "TPGGenerator"; // public String generator = "OCGenerator"; @Parameter(names = {"--totalEvents"}, description = "Total number of events to process.") - public int totalEvents = 48000; + public int totalEvents = 10000; @Parameter(names = {"--deposit_ratio"}, description = "Ratio of deposit for SL.") public Integer Ratio_Of_Deposit = 25; @@ -242,27 +254,41 @@ public void initializeCfg(HashMap config) { switch(application) { case "StreamLedger" : workloadType = "0"; - bottomLine = "300,3000,500,4000,0.2,0.2";//TD,LD,PD,SUM,VDD,R_of_A + bottomLine = "300,3000,500,1200,0.2,0.2";//TD,LD,PD,SUM,VDD,R_of_A schedulerPools = "OP_BFS_A,OP_NS_A,OG_NS_A"; defaultScheduler = "OP_BFS_A"; + phaseNum = shiftRate * 8; break; case "OnlineBiding" : workloadType ="1"; bottomLine = "500,5000,1,6000,0.2,0.2";//TD,LD,PD,SUM,VDD,R_of_A schedulerPools = "OG_BFS_A,OG_NS_A,OG_NS"; defaultScheduler = "OG_BFS_A"; + phaseNum = shiftRate * 3; break; case "GrepSum" : workloadType ="2"; bottomLine = "500,5000,6500,3000,0.2,0.2";//TD,LD,PD,SUM,VDD,R_of_A schedulerPools = "OP_NS_A,OG_BFS_A,OP_NS,OP_NS_A"; defaultScheduler = "OP_NS_A"; + phaseNum = shiftRate * 4; + break; + case "TollProcessing" : + workloadType ="3"; + phaseNum = shiftRate * 1; + defaultScheduler = "OG_BFS_A"; + SchedulersForGroup = "OP_BFS_A,OP_NS_A"; break; default: throw new IllegalStateException("Unexpected value: " + application); } /* Dynamic switch scheduler*/ - config.put("isDynamic",isDynamic); + if (isDynamic == 1) { + config.put("isDynamic", true); + config.put("totalEvents",phaseNum * tthread * checkpoint_interval); + } else { + config.put("isDynamic", false); + } config.put("schedulersPool",schedulerPools); config.put("defaultScheduler",defaultScheduler); config.put("scheduler", defaultScheduler); @@ -273,6 +299,17 @@ public void initializeCfg(HashMap config) { /* Dynamic Workload Configuration*/ config.put("workloadType",workloadType); config.put("shiftRate",shiftRate); + config.put("phaseNum",phaseNum); + + /* Group scheduler*/ + if (isGroup == 1) { + config.put("isGroup", true); + config.put("groupNum",groupNum); + config.put("SchedulersForGroup",SchedulersForGroup); + config.put("totalEvents",phaseNum * tthread * checkpoint_interval); + } else { + config.put("isGroup", false); + } System.setProperty("my.log", metric_path); } diff --git a/common/src/main/java/common/platform/Platform.java b/common/src/main/java/common/platform/Platform.java index 56e2b0a09..1146f2e93 100644 --- a/common/src/main/java/common/platform/Platform.java +++ b/common/src/main/java/common/platform/Platform.java @@ -92,7 +92,7 @@ public static ArrayList[] getNodes(int machine) { node_5, node_6, node_7}; - } else { + } else if (machine == 3){ // //a simple single-socket 48-core machine. // int cores = 48; // Integer[] no_0 = new Integer[cores]; @@ -110,6 +110,16 @@ public static ArrayList[] getNodes(int machine) { return new ArrayList[]{ node_0, node_1}; + } else { + //a two-sockets 40-core machine + //Intel(R) Xeon(R) Gold 6230 CPU @ 2.10GHz + Integer[] no_0 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}; + node_0 = new ArrayList<>(Arrays.asList(no_0)); + Integer[] no_1 = {20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39}; + ArrayList node_1 = new ArrayList<>(Arrays.asList(no_1)); + return new ArrayList[]{ + node_0, + node_1}; } } } \ No newline at end of file diff --git a/state-engine/src/main/java/profiler/MeasureTools.java b/state-engine/src/main/java/profiler/MeasureTools.java index 012d8961f..e4f56cb9e 100644 --- a/state-engine/src/main/java/profiler/MeasureTools.java +++ b/state-engine/src/main/java/profiler/MeasureTools.java @@ -8,6 +8,8 @@ import java.io.File; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import static common.CONTROL.enable_debug; @@ -362,10 +364,34 @@ private static void WriteThroughputReport(File file, double throughput) { e.printStackTrace(); } } + private static void WriteThroughputReportRuntime(File file, int tthread, int phase) { + try { + double[] tr = new double[phase]; + for (int i = 0; i < phase; i++) { + tr[i] = 0; + for (int j = 0; j < tthread; j++){ + tr[i] = tr[i] + Metrics.Runtime.ThroughputPerPhase.get(j).get(i) * 1E6;// + } + } + BufferedWriter fileWriter = Files.newBufferedWriter(Paths.get(file.getPath()), APPEND); + fileWriter.write("phase_id\t throughput\n"); + for (int i = 0; i < phase; i++){ + String output = String.format("%d\t" + + "%-10.4f\t" + , i,tr[i] + ); + fileWriter.write(output + "\n"); + } + fileWriter.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } - public static void METRICS_REPORT(int ccOption, File file, int tthread, double throughput) { + public static void METRICS_REPORT(int ccOption, File file, int tthread, double throughput, int phase) { WriteThroughputReport(file, throughput); AverageTotalTimeBreakdownReport(file, tthread); + WriteThroughputReportRuntime(file,tthread,phase); if (ccOption == CCOption_MorphStream) {//extra info SchedulerTimeBreakdownReport(file, tthread); } else { diff --git a/state-engine/src/main/java/profiler/Metrics.java b/state-engine/src/main/java/profiler/Metrics.java index 46176c087..c92cfdc12 100644 --- a/state-engine/src/main/java/profiler/Metrics.java +++ b/state-engine/src/main/java/profiler/Metrics.java @@ -2,6 +2,11 @@ import org.apache.commons.math.stat.descriptive.DescriptiveStatistics; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import static content.common.CommonMetaTypes.kMaxThreadNum; public class Metrics { @@ -55,6 +60,8 @@ public static void RECORD_TIME(int thread_id, int number_events) { Total_Record.stream_total[thread_id].addValue(stream_total); Total_Record.txn_total[thread_id].addValue(txn_total); Total_Record.overhead_total[thread_id].addValue(total_time - stream_total - txn_total); + Runtime.ThroughputPerPhase.get(thread_id).add(1 / total_time); + Runtime.Start[thread_id]=0; } public static void COMPUTE_PRE_EXE_START_TIME(int thread_id) { @@ -275,6 +282,8 @@ static class Runtime { public static long[] Post = new long[kMaxThreadNum]; public static long[] TxnStart = new long[kMaxThreadNum]; public static long[] Txn = new long[kMaxThreadNum]; + //Runtime throughput per phase + public static HashMap> ThroughputPerPhase =new HashMap<>(); //USED ONLY BY TStream public static long[] PreTxnStart = new long[kMaxThreadNum]; public static long[] PreTxn = new long[kMaxThreadNum]; @@ -290,6 +299,7 @@ public static void Initialize() { Txn[i] = 0; PreTxnStart[i] = 0; PreTxn[i] = 0; + ThroughputPerPhase.put(i,new ArrayList<>()); } } } diff --git a/state-engine/src/main/java/scheduler/collector/Collector.java b/state-engine/src/main/java/scheduler/collector/Collector.java index 0c15963bf..d1927562b 100644 --- a/state-engine/src/main/java/scheduler/collector/Collector.java +++ b/state-engine/src/main/java/scheduler/collector/Collector.java @@ -126,7 +126,7 @@ public String getDecision(int threadId){ } } - public boolean timeToSwitch(long markId,int threadId){ + public boolean timeToSwitch(long markId,int threadId,String currentScheduler){ if(isRuntime){ //TODO:collect information runtime return false; @@ -136,7 +136,11 @@ public boolean timeToSwitch(long markId,int threadId){ long currentId = Long.parseLong(workloadConfig.get(workloadId)[workloadConfig.get(workloadId).length-1]); if (currentId <= markId+threadCount-threadId){ currentWorkload.put(threadId,currentWorkload.get(threadId)+1); - return true; + if(getDecision(threadId).equals(currentScheduler)){ + return false; + } else { + return true; + } } } return false; diff --git a/state-engine/src/main/java/scheduler/impl/IScheduler.java b/state-engine/src/main/java/scheduler/impl/IScheduler.java index ed8063e0f..26125ce3f 100644 --- a/state-engine/src/main/java/scheduler/impl/IScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/IScheduler.java @@ -23,4 +23,5 @@ public interface IScheduler { void start_evaluation(Context context, long mark_ID, int num_events); + void initTPG(int offset); } diff --git a/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java b/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java index 6df68db0d..10816ad3b 100644 --- a/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/OGScheduler.java @@ -41,6 +41,11 @@ protected OGScheduler(int totalThreads, int NUM_ITEMS, int app) { this.tpg = new TaskPrecedenceGraph<>(totalThreads, delta, NUM_ITEMS, app); } + @Override + public void initTPG(int offset) { + tpg.initTPG(offset); + } + /** * state to thread mapping * @@ -450,4 +455,5 @@ protected boolean isConflicted(Context context, OperationChain operationChain, O } return false; } + } diff --git a/state-engine/src/main/java/scheduler/impl/og/structured/OGSScheduler.java b/state-engine/src/main/java/scheduler/impl/og/structured/OGSScheduler.java index a5f2ecec7..fba911c48 100644 --- a/state-engine/src/main/java/scheduler/impl/og/structured/OGSScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/og/structured/OGSScheduler.java @@ -20,7 +20,7 @@ public void INITIALIZE(Context context) { int threadId = context.thisThreadId; // tpg.constructTPG(context); tpg.firstTimeExploreTPG(context); - SOURCE_CONTROL.getInstance().preStateAccessBarrier(threadId);//sync for all threads to come to this line to ensure chains are constructed for the current batch. + SOURCE_CONTROL.getInstance().exploreTPGBarrier(threadId);//sync for all threads to come to this line to ensure chains are constructed for the current batch. } protected void ProcessedToNextLevel(Context context) { diff --git a/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java b/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java index aa57b55a1..17b4e371a 100644 --- a/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java +++ b/state-engine/src/main/java/scheduler/impl/op/OPScheduler.java @@ -32,12 +32,15 @@ public abstract class OPScheduler impl private static final Logger log = LoggerFactory.getLogger(OPScheduler.class); public final int delta;//range of each partition. depends on the number of op in the stage. public final TaskPrecedenceGraph tpg; // TPG to be maintained in this global instance. - + public int groupNum; public OPScheduler(int totalThreads, int NUM_ITEMS, int app) { delta = (int) Math.ceil(NUM_ITEMS / (double) totalThreads); // Check id generation in DateGenerator. this.tpg = new TaskPrecedenceGraph<>(totalThreads, delta, NUM_ITEMS, app); } - + @Override + public void initTPG(int offset) { + tpg.initTPG(offset); + } /** * state to thread mapping * @@ -339,4 +342,5 @@ public void start_evaluation(Context context, long mark_ID, int num_events) { RESET(context);// // MeasureTools.SCHEDULE_TIME_RECORD(threadId, num_events); } + } diff --git a/state-engine/src/main/java/scheduler/struct/og/TableOCs.java b/state-engine/src/main/java/scheduler/struct/og/TableOCs.java index 972cdcf6d..acaf79856 100644 --- a/state-engine/src/main/java/scheduler/struct/og/TableOCs.java +++ b/state-engine/src/main/java/scheduler/struct/og/TableOCs.java @@ -8,10 +8,10 @@ public class TableOCs { public ConcurrentHashMap> threadOCsMap = new ConcurrentHashMap<>();//each op has a holder. - public TableOCs(Integer num_op) { + public TableOCs(Integer num_op,int offset) { int i; for (i = 0; i < num_op; i++) { - threadOCsMap.put(i, new Holder()); + threadOCsMap.put(i + offset, new Holder()); } } } diff --git a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java index 2b7155b2a..9daac1cd1 100644 --- a/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/og/TaskPrecedenceGraph.java @@ -9,6 +9,7 @@ import scheduler.context.og.OGSContext; import scheduler.context.og.OGSchedulerContext; import scheduler.struct.op.MetaTypes; +import transaction.TxnManager; import transaction.impl.ordered.MyList; import utils.AppConfig; import utils.SOURCE_CONTROL; @@ -86,16 +87,18 @@ public TaskPrecedenceGraph(int totalThreads, int delta, int NUM_ITEMS, int app) this.app = app; //create holder. operationChains = new ConcurrentHashMap<>(); + } + public void initTPG(int offset) { if (app == 0) {//GS - operationChains.put("MicroTable", new TableOCs<>(totalThreads)); + operationChains.put("MicroTable", new TableOCs<>(totalThreads,offset)); } else if (app == 1) {//SL - operationChains.put("accounts", new TableOCs<>(totalThreads)); - operationChains.put("bookEntries", new TableOCs<>(totalThreads)); + operationChains.put("accounts", new TableOCs<>(totalThreads,offset)); + operationChains.put("bookEntries", new TableOCs<>(totalThreads,offset)); } else if(app == 2) {//TP - operationChains.put("segment_speed",new TableOCs<>(totalThreads)); - operationChains.put("segment_cnt",new TableOCs<>(totalThreads)); + operationChains.put("segment_speed",new TableOCs<>(totalThreads,offset)); + operationChains.put("segment_cnt",new TableOCs<>(totalThreads,offset)); } else if (app == 3) { - operationChains.put("goods",new TableOCs<>(totalThreads)); + operationChains.put("goods",new TableOCs<>(totalThreads,offset)); } } diff --git a/state-engine/src/main/java/scheduler/struct/op/TableOCs.java b/state-engine/src/main/java/scheduler/struct/op/TableOCs.java index da13dfec2..f64900388 100644 --- a/state-engine/src/main/java/scheduler/struct/op/TableOCs.java +++ b/state-engine/src/main/java/scheduler/struct/op/TableOCs.java @@ -8,10 +8,10 @@ public class TableOCs { public ConcurrentHashMap threadOCsMap = new ConcurrentHashMap<>();//each op has a holder. - public TableOCs(Integer num_op) { + public TableOCs(Integer num_op,int offset) { int i; for (i = 0; i < num_op; i++) { - threadOCsMap.put(i, new Holder()); + threadOCsMap.put(i+offset, new Holder()); } } } diff --git a/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java b/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java index 6b799b1c3..3f19ae559 100644 --- a/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java +++ b/state-engine/src/main/java/scheduler/struct/op/TaskPrecedenceGraph.java @@ -43,17 +43,17 @@ public class TaskPrecedenceGraph { public final ConcurrentHashMap> threadToOCs; public void reset(Context context) { - if (app == 0) { - operationChains.get("MicroTable").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); - } else if (app == 1) { - operationChains.get("accounts").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); - operationChains.get("bookEntries").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); - } else if (app == 2) { - operationChains.get("segment_speed").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); - operationChains.get("segment_cnt").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); - } else if (app == 3) { - operationChains.get("goods").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); - } +// if (app == 0) { +// operationChains.get("MicroTable").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); +// } else if (app == 1) { +// operationChains.get("accounts").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); +// operationChains.get("bookEntries").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); +// } else if (app == 2) { +// operationChains.get("segment_speed").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); +// operationChains.get("segment_cnt").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); +// } else if (app == 3) { +// operationChains.get("goods").threadOCsMap.get(context.thisThreadId).holder_v1.clear(); +// } // threadToOCs.get(context.thisThreadId).clear(); // for (OperationChain oc : threadToOCs.get(context.thisThreadId)) { // oc.clear(); @@ -78,16 +78,18 @@ public TaskPrecedenceGraph(int totalThreads, int delta, int NUM_ITEMS, int app) this.app = app; //create holder. operationChains = new ConcurrentHashMap<>(); + } + public void initTPG(int offset) { if (app == 0) {//GS - operationChains.put("MicroTable", new TableOCs(totalThreads)); + operationChains.put("MicroTable", new TableOCs(totalThreads,offset)); } else if (app == 1) {//SL - operationChains.put("accounts", new TableOCs(totalThreads)); - operationChains.put("bookEntries", new TableOCs(totalThreads)); + operationChains.put("accounts", new TableOCs(totalThreads,offset)); + operationChains.put("bookEntries", new TableOCs(totalThreads,offset)); } else if (app == 2){//TP - operationChains.put("segment_speed",new TableOCs(totalThreads)); - operationChains.put("segment_cnt",new TableOCs(totalThreads)); + operationChains.put("segment_speed",new TableOCs(totalThreads,offset)); + operationChains.put("segment_cnt",new TableOCs(totalThreads,offset)); } else if (app == 3) {//OB - operationChains.put("goods",new TableOCs(totalThreads)); + operationChains.put("goods",new TableOCs(totalThreads,offset)); } } diff --git a/state-engine/src/main/java/transaction/TxnManager.java b/state-engine/src/main/java/transaction/TxnManager.java index 14bd4bb6e..16e0040c2 100644 --- a/state-engine/src/main/java/transaction/TxnManager.java +++ b/state-engine/src/main/java/transaction/TxnManager.java @@ -1,5 +1,6 @@ package transaction; +import org.apache.zookeeper.txn.Txn; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scheduler.collector.Collector; @@ -29,12 +30,35 @@ public abstract class TxnManager implements ITxnManager { private static final Logger log = LoggerFactory.getLogger(TxnManager.class); protected static IScheduler scheduler; + /** + * For dynamic workload + */ protected static HashMap schedulerPool; protected static boolean enableDynamic=false; - protected static Collector collector=new Collector(); protected static String currentSchedulerType; + /** + * Scheduler for multiple workload + */ + protected static HashMap schedulerByGroup; + protected static HashMap schedulerTypeByGroup; + public static boolean enableGroup = false; + public static int groupNum; + + public static void CreateSchedulerByGroup(String schedulerType, int threadCount,int numberOfStates, int app){ + schedulerByGroup = new HashMap<>(); + schedulerTypeByGroup = new HashMap<>(); + String[] scheduler = schedulerType.split(","); + for(int i=0;i iteration; @@ -28,13 +30,14 @@ public static SOURCE_CONTROL getInstance() { return ourInstance; } - public void config(int number_threads) { + public void config(int number_threads, int groupNum) { totalThreads = number_threads; startBarrier = new CyclicBarrier(number_threads); endBarrier = new CyclicBarrier(number_threads); finalEndBarrier = new CyclicBarrier(number_threads); - switchSchedulerBarrier=new CyclicBarrier(number_threads); - dLevelEndBarrier = new Phaser(number_threads); + switchSchedulerBarrier = new CyclicBarrier(number_threads); + exploreTPGBarrier = new CyclicBarrier(number_threads / groupNum); + dLevelEndBarrier = new Phaser(number_threads / groupNum); iteration = new HashMap<>(); for (int i = 0; i < number_threads; i++) { iteration.put(i, 0); @@ -54,6 +57,13 @@ public void preStateAccessBarrier(int threadId) { } } + public void exploreTPGBarrier(int threadId) { + try { + exploreTPGBarrier.await(); + } catch (Exception e) { + e.printStackTrace(); + } + } public void postStateAccessBarrier(int threadId) { try { endBarrier.await(); diff --git a/stream-engine/src/main/java/components/operators/api/TransactionalBolt.java b/stream-engine/src/main/java/components/operators/api/TransactionalBolt.java index c1697d3ab..4e8a4aa82 100644 --- a/stream-engine/src/main/java/components/operators/api/TransactionalBolt.java +++ b/stream-engine/src/main/java/components/operators/api/TransactionalBolt.java @@ -173,6 +173,6 @@ public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { COMPUTE_COMPLEXITY = Metrics.COMPUTE_COMPLEXITY; POST_COMPUTE_COMPLEXITY = Metrics.POST_COMPUTE_COMPLEXITY; //LOG.DEBUG("NUM_ACCESSES: " + NUM_ACCESSES + " theta:" + theta); - SOURCE_CONTROL.getInstance().config(tthread); + SOURCE_CONTROL.getInstance().config(tthread,config.getInt("groupNum")); } } \ No newline at end of file diff --git a/stream-engine/src/main/java/controller/affinity/SequentialBinding.java b/stream-engine/src/main/java/controller/affinity/SequentialBinding.java index a6e6d5016..f99dd106a 100644 --- a/stream-engine/src/main/java/controller/affinity/SequentialBinding.java +++ b/stream-engine/src/main/java/controller/affinity/SequentialBinding.java @@ -21,7 +21,7 @@ public static int next_cpu() { ArrayList list = mapping_node[socket]; Integer core = list.get(cpu); cpu++; - if (cpu == 24) {//assume one socket 24 cores. + if (cpu == 19) {//assume one socket 24 cores. socket++; cpu = 0; } diff --git a/stream-engine/src/main/java/execution/ExecutionManager.java b/stream-engine/src/main/java/execution/ExecutionManager.java index 728f5f1cd..a94cb586a 100644 --- a/stream-engine/src/main/java/execution/ExecutionManager.java +++ b/stream-engine/src/main/java/execution/ExecutionManager.java @@ -74,6 +74,8 @@ public void distributeTasks(Configuration conf, if (!conf.getBoolean("isRuntime")) { TxnManager.setWorkloadConfig(conf.getString("WorkloadConfig")); } + } else if (conf.getBoolean("isGroup")) { + TxnManager.CreateSchedulerByGroup(conf.getString("SchedulersForGroup"),totalThread,numberOfStates,app); } else { TxnManager.CreateScheduler(schedulerType, totalThread, numberOfStates, app); } From a1d1908fc734d901ce5d883c77d589b28e2e5236 Mon Sep 17 00:00:00 2001 From: curryzjj Date: Thu, 31 Mar 2022 22:30:35 +0800 Subject: [PATCH 15/15] Add Ratio_of_Multiple_State_Access_Ratio for GS --- affinity/src/main/c/Makefile | 4 ++-- .../apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java | 8 ++++++++ .../apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java | 2 ++ .../GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java | 2 ++ .../DynamicDataGeneratorConfig.java | 2 ++ .../topology/transactional/initializer/GSInitializer.java | 2 +- common/src/main/java/common/Runner.java | 4 ++++ state-engine/src/main/java/profiler/MeasureTools.java | 6 +++--- .../java/components/operators/api/TransactionalBolt.java | 6 +++++- 9 files changed, 29 insertions(+), 7 deletions(-) diff --git a/affinity/src/main/c/Makefile b/affinity/src/main/c/Makefile index 86e88454f..d45f02e3c 100755 --- a/affinity/src/main/c/Makefile +++ b/affinity/src/main/c/Makefile @@ -19,8 +19,8 @@ JNI_JAVA_SOURCES := $(patsubst %,$(TARGET_DIR)/%.class,$(subst .,/,$(JAVA_CLASSE JAVA_BUILD_DIR := $(TARGET_DIR) -#JAVA_HOME ?= /usr/java/default -JAVA_HOME :=/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home +JAVA_HOME ?= /usr/java/default +#JAVA_HOME :=/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home JAVA_LIB := $(JAVA_HOME)/jre/lib JVM_SHARED_LIBS := -L$(JAVA_LIB)/amd64/server -L$(JAVA_LIB)/i386/server -L$(JAVA_LIB)/amd64/jrockit/ -L$(JAVA_LIB)/i386/jrockit/ -L$(JAVA_LIB)/ppc64le/server -L$(JAVA_LIB)/ppc64le/jrockit/ diff --git a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java index e619e35a3..9a255c939 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java @@ -39,6 +39,7 @@ public class GSTPGDataGenerator extends DataGenerator { private final int State_Access_Skewness; // ratio of state access, following zipf distribution private final int Ratio_of_Transaction_Aborts; // ratio of transaction aborts, fail the transaction or not. i.e. transfer amount might be invalid. private final int Ratio_of_Overlapped_Keys; // ratio of overlapped keys in transactions, which affects the dependencies and circulars. + private final int Ratio_of_Multiple_State_Access_Ratio;//ratio of multiple state access per transaction private final int Transaction_Length; // control the number of txns overlap with each other. private final ArrayList generatedKeys = new ArrayList<>(); @@ -66,6 +67,7 @@ public GSTPGDataGenerator(GSTPGDataGeneratorConfig dataConfig) { Ratio_of_Transaction_Aborts = dataConfig.Ratio_of_Transaction_Aborts; Ratio_of_Overlapped_Keys = dataConfig.Ratio_of_Overlapped_Keys; Transaction_Length = dataConfig.Transaction_Length; + Ratio_of_Multiple_State_Access_Ratio = dataConfig.Ratio_of_Multiple_State_Access_Ratio; int nKeyState = dataConfig.getnKeyStates(); @@ -104,6 +106,12 @@ protected void generateTuple() { } private GSEvent randomEvent() { + int NUM_ACCESS; + if (random.nextInt(100) < Ratio_of_Multiple_State_Access_Ratio) { + NUM_ACCESS = this.NUM_ACCESS; + } else { + NUM_ACCESS = 1; + } int[] keys = new int[NUM_ACCESS*Transaction_Length]; int writeLevel = -1; if (!isUnique) { diff --git a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java index d716a0dde..4fcf75484 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java +++ b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java @@ -16,6 +16,7 @@ public class GSTPGDataGeneratorConfig extends DataGeneratorConfig { public int Ratio_of_Overlapped_Keys; public int Ratio_of_Transaction_Aborts; public int Transaction_Length; + public int Ratio_of_Multiple_State_Access_Ratio; @Override public void initialize(Configuration config) { @@ -25,5 +26,6 @@ public void initialize(Configuration config) { Ratio_of_Overlapped_Keys = config.getInt("Ratio_of_Overlapped_Keys", 0); Ratio_of_Transaction_Aborts = config.getInt("Ratio_of_Transaction_Aborts", 0); Transaction_Length = config.getInt("Transaction_Length", 1); + Ratio_of_Multiple_State_Access_Ratio = config.getInt("Ratio_of_Multiple_State_Access_Ratio",100); } } \ No newline at end of file diff --git a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java index 8444ba3bc..f206247c8 100644 --- a/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java +++ b/application/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java @@ -26,6 +26,7 @@ public class GSTPGDynamicDataGenerator extends DynamicWorkloadGenerator { private int Ratio_of_Transaction_Aborts; // ratio of transaction aborts, fail the transaction or not. i.e. transfer amount might be invalid. private int Ratio_of_Overlapped_Keys; // ratio of overlapped keys in transactions, which affects the dependencies and circulars. private int Transaction_Length; + private int Ratio_of_Multiple_State_Access_Ratio;//ratio of multiple state access per transaction private int tthread; private int nKeyState; // control the number of txns overlap with each other. @@ -91,6 +92,7 @@ public void switchConfiguration(String type) { Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts; Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys; Transaction_Length = dynamicDataConfig.Transaction_Length; + Ratio_of_Multiple_State_Access_Ratio = dynamicDataConfig.Ratio_of_Multiple_State_Access_Ratio; nKeyState = dynamicDataConfig.getnKeyStates(); int MAX_LEVEL = 256; diff --git a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java index 28e7b2327..58f320162 100644 --- a/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java +++ b/application/src/main/java/benchmark/dynamicWorkloadGenerator/DynamicDataGeneratorConfig.java @@ -27,6 +27,7 @@ public class DynamicDataGeneratorConfig extends DataGeneratorConfig { public int Ratio_Of_Deposit; public int Ratio_Of_Buying; public boolean enableGroup; + public int Ratio_of_Multiple_State_Access_Ratio; public void initialize(Configuration config) { super.initialize(config); @@ -42,6 +43,7 @@ public void initialize(Configuration config) { Transaction_Length = config.getInt("Transaction_Length", 1); Ratio_Of_Deposit = config.getInt("Ratio_Of_Deposit", 0); Ratio_Of_Buying=config.getInt("Ratio_Of_Buying",0); + Ratio_of_Multiple_State_Access_Ratio = config.getInt("Ratio_of_Multiple_State_Access_Ratio",100); phase = 0; } diff --git a/application/src/main/java/common/topology/transactional/initializer/GSInitializer.java b/application/src/main/java/common/topology/transactional/initializer/GSInitializer.java index b00331514..7d3db4462 100644 --- a/application/src/main/java/common/topology/transactional/initializer/GSInitializer.java +++ b/application/src/main/java/common/topology/transactional/initializer/GSInitializer.java @@ -301,7 +301,7 @@ private void loadMicroEvents(BufferedReader reader, int totalEvents, boolean shu String[] split = txn.split(","); int npid = (int) (Long.parseLong(split[1]) / partitionOffset); // construct bid array - int keyLength = NUM_ACCESS*Transaction_Length; + int keyLength = split.length - 2; HashMap pids = new HashMap<>(); long[] keys = new long[keyLength]; for (int i = 1; i < keyLength+1; i++) { diff --git a/common/src/main/java/common/Runner.java b/common/src/main/java/common/Runner.java index 88c24fd71..12ddef274 100644 --- a/common/src/main/java/common/Runner.java +++ b/common/src/main/java/common/Runner.java @@ -159,6 +159,9 @@ public abstract class Runner implements IRunner { @Parameter(names = {"--key_skewness"}, description = "State access skewness.") public Integer State_Access_Skewness = 0; + @Parameter(names = {"--multiple_ratio"}, description = "State access skewness.") + public Integer Ratio_of_Multiple_State_Access_Ratio = 100; + @Parameter(names = {"--overlap_ratio"}, description = "Ratio of overlapped keys.") public Integer Ratio_of_Overlapped_Keys = 10; @@ -211,6 +214,7 @@ public void initializeCfg(HashMap config) { config.put("State_Access_Skewness", State_Access_Skewness); config.put("Ratio_of_Overlapped_Keys", Ratio_of_Overlapped_Keys); config.put("Ratio_of_Transaction_Aborts", Ratio_of_Transaction_Aborts); + config.put("Ratio_of_Multiple_State_Access_Ratio",Ratio_of_Multiple_State_Access_Ratio); config.put("Transaction_Length", Transaction_Length); config.put("numberOfDLevels", numberOfDLevels); diff --git a/state-engine/src/main/java/profiler/MeasureTools.java b/state-engine/src/main/java/profiler/MeasureTools.java index e4f56cb9e..c25417b96 100644 --- a/state-engine/src/main/java/profiler/MeasureTools.java +++ b/state-engine/src/main/java/profiler/MeasureTools.java @@ -366,8 +366,8 @@ private static void WriteThroughputReport(File file, double throughput) { } private static void WriteThroughputReportRuntime(File file, int tthread, int phase) { try { - double[] tr = new double[phase]; - for (int i = 0; i < phase; i++) { + double[] tr = new double[Metrics.Runtime.ThroughputPerPhase.get(0).size()]; + for (int i = 0; i < tr.length; i++) { tr[i] = 0; for (int j = 0; j < tthread; j++){ tr[i] = tr[i] + Metrics.Runtime.ThroughputPerPhase.get(j).get(i) * 1E6;// @@ -375,7 +375,7 @@ private static void WriteThroughputReportRuntime(File file, int tthread, int pha } BufferedWriter fileWriter = Files.newBufferedWriter(Paths.get(file.getPath()), APPEND); fileWriter.write("phase_id\t throughput\n"); - for (int i = 0; i < phase; i++){ + for (int i = 0; i < tr.length; i++){ String output = String.format("%d\t" + "%-10.4f\t" , i,tr[i] diff --git a/stream-engine/src/main/java/components/operators/api/TransactionalBolt.java b/stream-engine/src/main/java/components/operators/api/TransactionalBolt.java index 4e8a4aa82..59d71fbac 100644 --- a/stream-engine/src/main/java/components/operators/api/TransactionalBolt.java +++ b/stream-engine/src/main/java/components/operators/api/TransactionalBolt.java @@ -173,6 +173,10 @@ public void initialize(int thread_Id, int thisTaskId, ExecutionGraph graph) { COMPUTE_COMPLEXITY = Metrics.COMPUTE_COMPLEXITY; POST_COMPUTE_COMPLEXITY = Metrics.POST_COMPUTE_COMPLEXITY; //LOG.DEBUG("NUM_ACCESSES: " + NUM_ACCESSES + " theta:" + theta); - SOURCE_CONTROL.getInstance().config(tthread,config.getInt("groupNum")); + if (config.getBoolean("isGroup")) { + SOURCE_CONTROL.getInstance().config(tthread,config.getInt("groupNum")); + } else { + SOURCE_CONTROL.getInstance().config(tthread,1); + } } } \ No newline at end of file