Skip to content

dynamic and group workload #97

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
curryzjj wants to merge 20 commits into from
Closed

dynamic and group workload #97

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
8cfaa77
Add Collector
curryzjj Mar 11, 2022
719dba9
Merge branch 'intellistream:main' into main
curryzjj Mar 11, 2022
fbe53b8
Add application Bolt(TP,OB) finish dynamic scheduler
curryzjj Mar 12, 2022
6346d6a
Finish dynamic scheduler
curryzjj Mar 14, 2022
91db916
Only the thread-0
curryzjj Mar 14, 2022
e03880f
Fix concurrent access bug
curryzjj Mar 14, 2022
4bab13f
Add a configuration
curryzjj Mar 15, 2022
19f9220
System sensitivity study (#86)
yancanmao Mar 13, 2022
2aedfbe
solve conflict
curryzjj Mar 14, 2022
38c1002
Solve the conflict
curryzjj Mar 15, 2022
027197e
Merge branch 'main' into SchedulerPool
curryzjj Mar 15, 2022
e7f0f71
Merge branch 'intellistream:main' into SchedulerPool
curryzjj Mar 16, 2022
4678f24
Add DynamicWorkloadGenerator
curryzjj Mar 16, 2022
ba60f1a
Add Application(TP and OB)
curryzjj Mar 18, 2022
281a663
Fix some typo
curryzjj Mar 18, 2022
f96f44e
Merge branch 'intellistream:main' into SchedulerPool
curryzjj Mar 29, 2022
4fbc04c
Workload configuration per phase
curryzjj Mar 29, 2022
3147aa1
Merge branch 'intellistream:main' into SchedulerPool
curryzjj Mar 31, 2022
79a9ac6
Add Groups for TP
curryzjj Mar 31, 2022
a1d1908
Add Ratio_of_Multiple_State_Access_Ratio for GS
curryzjj Mar 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions affinity/src/main/c/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ JNI_JAVA_SOURCES := $(patsubst %,$(TARGET_DIR)/%.class,$(subst .,/,$(JAVA_CLASSE
JAVA_BUILD_DIR := $(TARGET_DIR)

JAVA_HOME ?= /usr/java/default
#JAVA_HOME :=/Library/Java/JavaVirtualMachines/jdk1.8.0_191.jdk/Contents/Home
#JAVA_HOME :=/Library/Java/JavaVirtualMachines/jdk1.8.0_231.jdk/Contents/Home
JAVA_LIB := $(JAVA_HOME)/jre/lib
JVM_SHARED_LIBS := -L$(JAVA_LIB)/amd64/server -L$(JAVA_LIB)/i386/server -L$(JAVA_LIB)/amd64/jrockit/ -L$(JAVA_LIB)/i386/jrockit/ -L$(JAVA_LIB)/ppc64le/server -L$(JAVA_LIB)/ppc64le/jrockit/

CXX=g++
INCLUDES := -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(WORKING_DIR)
INCLUDES := -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(WORKING_DIR) -I $(JAVA_HOME)/include/darwin

# classpath for javah
ifdef CLASSPATH
Expand Down
5 changes: 5 additions & 0 deletions application/src/main/java/benchmark/datagenerator/DataGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;

/**
* Data generator for benchmarks, this class contains all common methods and attributes that can be used in each application
*/
Expand Down Expand Up @@ -45,5 +47,8 @@ public void clearDataStructures() {
}
/* Switch configuration, used in the dynamic data generator*/

public List<String> getTranToDecisionConf() {
return null;
}

}
8 changes: 8 additions & 0 deletions ...ion/src/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public class GSTPGDataGenerator extends DataGenerator {
private final int State_Access_Skewness; // ratio of state access, following zipf distribution
private final int Ratio_of_Transaction_Aborts; // ratio of transaction aborts, fail the transaction or not. i.e. transfer amount might be invalid.
private final int Ratio_of_Overlapped_Keys; // ratio of overlapped keys in transactions, which affects the dependencies and circulars.
private final int Ratio_of_Multiple_State_Access_Ratio;//ratio of multiple state access per transaction
private final int Transaction_Length;
// control the number of txns overlap with each other.
private final ArrayList<Integer> generatedKeys = new ArrayList<>();
Expand Down Expand Up @@ -66,6 +67,7 @@ public GSTPGDataGenerator(GSTPGDataGeneratorConfig dataConfig) {
Ratio_of_Transaction_Aborts = dataConfig.Ratio_of_Transaction_Aborts;
Ratio_of_Overlapped_Keys = dataConfig.Ratio_of_Overlapped_Keys;
Transaction_Length = dataConfig.Transaction_Length;
Ratio_of_Multiple_State_Access_Ratio = dataConfig.Ratio_of_Multiple_State_Access_Ratio;

int nKeyState = dataConfig.getnKeyStates();

Expand Down Expand Up @@ -104,6 +106,12 @@ protected void generateTuple() {
}

private GSEvent randomEvent() {
int NUM_ACCESS;
if (random.nextInt(100) < Ratio_of_Multiple_State_Access_Ratio) {
NUM_ACCESS = this.NUM_ACCESS;
} else {
NUM_ACCESS = 1;
}
int[] keys = new int[NUM_ACCESS*Transaction_Length];
int writeLevel = -1;
if (!isUnique) {
Expand Down
2 changes: 2 additions & 0 deletions ...c/main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDataGeneratorConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public class GSTPGDataGeneratorConfig extends DataGeneratorConfig {
public int Ratio_of_Overlapped_Keys;
public int Ratio_of_Transaction_Aborts;
public int Transaction_Length;
public int Ratio_of_Multiple_State_Access_Ratio;

@Override
public void initialize(Configuration config) {
Expand All @@ -25,5 +26,6 @@ public void initialize(Configuration config) {
Ratio_of_Overlapped_Keys = config.getInt("Ratio_of_Overlapped_Keys", 0);
Ratio_of_Transaction_Aborts = config.getInt("Ratio_of_Transaction_Aborts", 0);
Transaction_Length = config.getInt("Transaction_Length", 1);
Ratio_of_Multiple_State_Access_Ratio = config.getInt("Ratio_of_Multiple_State_Access_Ratio",100);
}
}
87 changes: 73 additions & 14 deletions .../main/java/benchmark/datagenerator/apps/GS/TPGTxnGenerator/GSTPGDynamicDataGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ public class GSTPGDynamicDataGenerator extends DynamicWorkloadGenerator {
private int Ratio_of_Transaction_Aborts; // ratio of transaction aborts, fail the transaction or not. i.e. transfer amount might be invalid.
private int Ratio_of_Overlapped_Keys; // ratio of overlapped keys in transactions, which affects the dependencies and circulars.
private int Transaction_Length;
private int Ratio_of_Multiple_State_Access_Ratio;//ratio of multiple state access per transaction
private int tthread;
private int nKeyState;
// control the number of txns overlap with each other.
private ArrayList<Integer> generatedKeys = new ArrayList<>();
// independent transactions.
Expand All @@ -49,21 +51,78 @@ public GSTPGDynamicDataGenerator(DynamicDataGeneratorConfig dynamicDataConfig) {
}

@Override
public void switchConfiguration() {
State_Access_Skewness = dynamicDataConfig.State_Access_Skewness;
NUM_ACCESS = dynamicDataConfig.NUM_ACCESS;
Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts;
Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys;
Transaction_Length = dynamicDataConfig.Transaction_Length;

int nKeyState = dynamicDataConfig.getnKeyStates();
int MAX_LEVEL = 256;
for (int i = 0; i < nKeyState; i++) {
idToLevel.put(i, i% MAX_LEVEL);
public void tranToDecisionConf() {
//TD,LD,PD,VDD,R_of_A,isCD,isCC,
StringBuilder stringBuilder = new StringBuilder();
//TODO:hard code, function not sure
double td = Transaction_Length * dynamicDataConfig.getCheckpoint_interval();
td = td *((double) Ratio_of_Overlapped_Keys/100);
stringBuilder.append(td);
stringBuilder.append(",");
double ld = Transaction_Length * dynamicDataConfig.getCheckpoint_interval();
stringBuilder.append(ld);
stringBuilder.append(",");
double pd = Transaction_Length * dynamicDataConfig.getCheckpoint_interval() * ((double) Ratio_of_Overlapped_Keys/100) * NUM_ACCESS;
stringBuilder.append(pd);
stringBuilder.append(",");
stringBuilder.append((double) State_Access_Skewness/100);
stringBuilder.append(",");
stringBuilder.append((double) Ratio_of_Transaction_Aborts/10000);
stringBuilder.append(",");
if (AppConfig.isCyclic) {
stringBuilder.append("1,");
} else {
stringBuilder.append("0,");
}
if (AppConfig.complexity < 40000){
stringBuilder.append("0,");
} else {
stringBuilder.append("1,");
}
stringBuilder.append(eventID+dynamicDataConfig.getShiftRate()*dynamicDataConfig.getCheckpoint_interval()*dynamicDataConfig.getTotalThreads());
this.tranToDecisionConf.add(stringBuilder.toString());
}

@Override
public void switchConfiguration(String type) {
switch (type) {
case "default" :
State_Access_Skewness = dynamicDataConfig.State_Access_Skewness;
NUM_ACCESS = dynamicDataConfig.NUM_ACCESS;
Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts;
Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys;
Transaction_Length = dynamicDataConfig.Transaction_Length;
Ratio_of_Multiple_State_Access_Ratio = dynamicDataConfig.Ratio_of_Multiple_State_Access_Ratio;

nKeyState = dynamicDataConfig.getnKeyStates();
int MAX_LEVEL = 256;
for (int i = 0; i < nKeyState; i++) {
idToLevel.put(i, i% MAX_LEVEL);
}
keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678);
configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState);
p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0);
break;
case "LD" :
Transaction_Length = dynamicDataConfig.Transaction_Length;
break;
case "isCyclic" :
Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts;
State_Access_Skewness = dynamicDataConfig.State_Access_Skewness;
keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678);
configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState);
p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0);
AppConfig.isCyclic = true;
break;
case "complexity" :
AppConfig.complexity = 80000;
break;
case "unchanging" :
break;
default:
throw new IllegalStateException("Unexpected value: " + type);
}
keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678);
configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState);
p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0);
tranToDecisionConf();
}

@Override
Expand Down
82 changes: 68 additions & 14 deletions application/src/main/java/benchmark/datagenerator/apps/OB/OBTPGDynamicDataGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ public class OBTPGDynamicDataGenerator extends DynamicWorkloadGenerator {
private int Transaction_Length;
private int tthread;
private int Ratio_of_Buying;
private int nKeyState;
// control the number of txns overlap with each other.
private ArrayList<Integer> generatedKeys = new ArrayList<>();
// independent transactions.
Expand Down Expand Up @@ -67,24 +68,77 @@ protected void generateTuple() {
}
events.add(event);
}
@Override
public void tranToDecisionConf() {
//TD,LD,PD,VDD,R_of_A,isCD,isCC,
StringBuilder stringBuilder = new StringBuilder();
//TODO:hard code, function not sure
double td = Transaction_Length * dynamicDataConfig.getCheckpoint_interval() * (1 - (double)Ratio_of_Buying/100) +
((double)Ratio_of_Buying/100) * 1 * dynamicDataConfig.getCheckpoint_interval();
td = td *((double) Ratio_of_Overlapped_Keys/100);
stringBuilder.append(td);
stringBuilder.append(",");
double ld = Transaction_Length * dynamicDataConfig.getCheckpoint_interval() * (1 - (double)Ratio_of_Buying/100) +
((double)Ratio_of_Buying/100) * 1 * dynamicDataConfig.getCheckpoint_interval();
stringBuilder.append(ld);
stringBuilder.append(",");
double pd = 0;
stringBuilder.append(pd);
stringBuilder.append(",");
stringBuilder.append((double) State_Access_Skewness/100);
stringBuilder.append(",");
stringBuilder.append((double) Ratio_of_Transaction_Aborts/10000);
stringBuilder.append(",");
if (AppConfig.isCyclic) {
stringBuilder.append("1,");
} else {
stringBuilder.append("0,");
}
if (AppConfig.complexity < 40000){
stringBuilder.append("0,");
} else {
stringBuilder.append("1,");
}
stringBuilder.append(eventID+dynamicDataConfig.getShiftRate()*dynamicDataConfig.getCheckpoint_interval()*dynamicDataConfig.getTotalThreads());
this.tranToDecisionConf.add(stringBuilder.toString());
}

@Override
public void switchConfiguration() {
State_Access_Skewness = dynamicDataConfig.State_Access_Skewness;
NUM_ACCESS = dynamicDataConfig.NUM_ACCESS;
Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts;
Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys;
Transaction_Length = dynamicDataConfig.Transaction_Length;
Ratio_of_Buying=dynamicDataConfig.Ratio_Of_Buying;
public void switchConfiguration(String type) {
switch (type){
case "default" :
State_Access_Skewness = dynamicDataConfig.State_Access_Skewness;
NUM_ACCESS = 1;
Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts;
Ratio_of_Overlapped_Keys = dynamicDataConfig.Ratio_of_Overlapped_Keys;
Transaction_Length = dynamicDataConfig.Transaction_Length;
Ratio_of_Buying=dynamicDataConfig.Ratio_Of_Buying;

int nKeyState = dynamicDataConfig.getnKeyStates();
int MAX_LEVEL = 256;
for (int i = 0; i < nKeyState; i++) {
idToLevel.put(i, i% MAX_LEVEL);
nKeyState = dynamicDataConfig.getnKeyStates();
int MAX_LEVEL = 256;
for (int i = 0; i < nKeyState; i++) {
idToLevel.put(i, i% MAX_LEVEL);
}
keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678);
configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState);
p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0);
break;
case "skew" :
State_Access_Skewness = dynamicDataConfig.State_Access_Skewness;
nKeyState = dynamicDataConfig.getnKeyStates();
keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678);
configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState);
p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0);
break;
case "abort" :
Ratio_of_Transaction_Aborts = dynamicDataConfig.Ratio_of_Transaction_Aborts;
break;
case "unchanging" :
break;
default:
throw new IllegalStateException("Unexpected value: " + type);
}
keyZipf = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0, 12345678);
configure_store(1, (double) State_Access_Skewness / 100, dynamicDataConfig.getTotalThreads(), nKeyState);
p_generator = new FastZipfGenerator(nKeyState, (double) State_Access_Skewness / 100, 0);
tranToDecisionConf();
}

@Override
Expand Down
Loading