From 57f8bac308d3a2d8d0ce869eb79599ea5e048ce3 Mon Sep 17 00:00:00 2001 From: Boris S Date: Thu, 26 Sep 2019 11:02:54 -0700 Subject: [PATCH 1/9] rename --- .../system/kinesis/KinesisSystemFactory.java | 4 +- .../samza/system/eventhub/EventHubConfig.java | 4 +- .../producer/AsyncSystemProducer.java | 4 +- .../samza/config/DefaultChooserConfig.java | 4 +- .../org/apache/samza/config/SystemConfig.java | 4 +- .../samza/execution/ExecutionPlanner.java | 6 +- .../JobNodeConfigurationGenerator.java | 12 +- .../apache/samza/execution/JobPlanner.java | 4 +- .../apache/samza/execution/StreamEdge.java | 18 +- .../apache/samza/execution/StreamManager.java | 2 +- .../operators/impl/OperatorImplGraph.java | 10 +- .../samza/storage/StorageManagerUtil.java | 4 +- .../storage/TaskSideInputStorageManager.java | 4 +- .../org/apache/samza/util/StreamUtil.java | 8 +- .../samza/checkpoint/OffsetManager.scala | 2 +- ...StreamConfig.scala => StreamConfig1.scala} | 52 +- .../samza/container/SamzaContainer.scala | 2 +- .../apache/samza/container/TaskInstance.scala | 2 +- .../org/apache/samza/job/JobRunner.scala | 2 +- .../MetricsSnapshotReporterFactory.scala | 2 +- .../apache/samza/config/TestStreamConfig.java | 460 +++++++++--------- .../apache/samza/config/TestSystemConfig.java | 20 +- .../samza/execution/TestExecutionPlanner.java | 4 +- .../samza/execution/TestStreamEdge.java | 8 +- .../operators/impl/TestOperatorImplGraph.java | 4 +- .../samza/testUtils/StreamTestUtils.java | 6 +- .../org/apache/samza/util/TestStreamUtil.java | 120 ++--- .../samza/system/kafka/KafkaSystemAdmin.java | 4 +- .../system/kafka/KafkaSystemFactory.scala | 2 +- .../samza/config/Log4jSystemConfig.java | 2 +- .../samza/config/Log4jSystemConfig.java | 2 +- .../samza/sql/interfaces/SqlIOConfig.java | 8 +- .../sql/translator/TestQueryTranslator.java | 20 +- .../samza/test/framework/TestRunner.java | 8 +- .../samza/test/operator/TestAsyncFlatMap.java | 4 +- .../TestLocalTableWithSideInputsEndToEnd.java | 8 +- 36 files changed, 415 insertions(+), 415 deletions(-) rename samza-core/src/main/scala/org/apache/samza/config/{StreamConfig.scala => StreamConfig1.scala} (89%) diff --git a/samza-aws/src/main/java/org/apache/samza/system/kinesis/KinesisSystemFactory.java b/samza-aws/src/main/java/org/apache/samza/system/kinesis/KinesisSystemFactory.java index 0086643b6b..dc7c3c71fe 100644 --- a/samza-aws/src/main/java/org/apache/samza/system/kinesis/KinesisSystemFactory.java +++ b/samza-aws/src/main/java/org/apache/samza/system/kinesis/KinesisSystemFactory.java @@ -22,7 +22,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.ConfigException; import org.apache.samza.config.JobConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.config.TaskConfig; import org.apache.samza.container.grouper.stream.AllSspToSingleTaskGrouperFactory; import org.apache.samza.metrics.MetricsRegistry; @@ -77,7 +77,7 @@ protected void validateConfig(String system, Config config) { // Kinesis streams cannot be configured as bootstrap streams KinesisConfig kConfig = new KinesisConfig(config); kConfig.getKinesisStreams(system).forEach(stream -> { - StreamConfig streamConfig = new StreamConfig(kConfig); + StreamConfig1 streamConfig = new StreamConfig1(kConfig); SystemStream ss = new SystemStream(system, stream); if (streamConfig.getBootstrapEnabled(ss)) { throw new ConfigException("Kinesis streams cannot be configured as bootstrap streams."); diff --git a/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java b/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java index d6cdab560f..d6f67db63e 100644 --- a/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java +++ b/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java @@ -24,7 +24,7 @@ import org.apache.samza.SamzaException; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.system.eventhub.producer.EventHubSystemProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,7 +106,7 @@ public EventHubConfig(Config config) { super(config); // Build reverse index for streamName -> streamId - StreamConfig streamConfig = new StreamConfig(config); + StreamConfig1 streamConfig = new StreamConfig1(config); LOG.info("Building mappings from physicalName to streamId"); JavaConversions.asJavaCollection(streamConfig.getStreamIds()) diff --git a/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/AsyncSystemProducer.java b/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/AsyncSystemProducer.java index 83d51ede92..c0bd44eb08 100644 --- a/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/AsyncSystemProducer.java +++ b/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/AsyncSystemProducer.java @@ -34,7 +34,7 @@ import java.util.stream.Collectors; import org.apache.samza.SamzaException; import org.apache.samza.config.Config; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.metrics.Counter; import org.apache.samza.metrics.MetricsRegistry; import org.apache.samza.system.OutgoingMessageEnvelope; @@ -109,7 +109,7 @@ public abstract class AsyncSystemProducer implements SystemProducer { * @param metricsRegistry the registry */ public AsyncSystemProducer(String systemName, Config config, MetricsRegistry metricsRegistry) { - StreamConfig sconfig = new StreamConfig(config); + StreamConfig1 sconfig = new StreamConfig1(config); streamIds = config.getList(String.format(CONFIG_STREAM_LIST, systemName)); physicalToStreamIds = streamIds.stream().collect(Collectors.toMap(sconfig::getPhysicalName, Function.identity())); diff --git a/samza-core/src/main/java/org/apache/samza/config/DefaultChooserConfig.java b/samza-core/src/main/java/org/apache/samza/config/DefaultChooserConfig.java index c753b4e10a..ebf9530f62 100644 --- a/samza-core/src/main/java/org/apache/samza/config/DefaultChooserConfig.java +++ b/samza-core/src/main/java/org/apache/samza/config/DefaultChooserConfig.java @@ -33,12 +33,12 @@ public class DefaultChooserConfig extends MapConfig { private static final String BATCH_SIZE = "task.consumer.batch.size"; private final TaskConfig taskConfig; - private final StreamConfig streamConfig; + private final StreamConfig1 streamConfig; public DefaultChooserConfig(Config config) { super(config); taskConfig = new TaskConfig(config); - streamConfig = new StreamConfig(config); + streamConfig = new StreamConfig1(config); } /** diff --git a/samza-core/src/main/java/org/apache/samza/config/SystemConfig.java b/samza-core/src/main/java/org/apache/samza/config/SystemConfig.java index 1403b0db17..7e3767b810 100644 --- a/samza-core/src/main/java/org/apache/samza/config/SystemConfig.java +++ b/samza-core/src/main/java/org/apache/samza/config/SystemConfig.java @@ -160,7 +160,7 @@ public String getSystemOffsetDefault(String systemName) { * @return the key serde for the {@code systemName}, or empty if it was not found */ public Optional getSystemKeySerde(String systemName) { - return getSystemDefaultStreamProperty(systemName, StreamConfig.KEY_SERDE()); + return getSystemDefaultStreamProperty(systemName, StreamConfig1.KEY_SERDE()); } /** @@ -168,7 +168,7 @@ public Optional getSystemKeySerde(String systemName) { * @return the message serde for the {@code systemName}, or empty if it was not found */ public Optional getSystemMsgSerde(String systemName) { - return getSystemDefaultStreamProperty(systemName, StreamConfig.MSG_SERDE()); + return getSystemDefaultStreamProperty(systemName, StreamConfig1.MSG_SERDE()); } /** diff --git a/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java b/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java index d59eb514bb..58ae1aa150 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java +++ b/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java @@ -44,7 +44,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.operators.spec.InputOperatorSpec; import org.apache.samza.operators.spec.OperatorSpec; import org.apache.samza.operators.spec.StreamTableJoinOperatorSpec; @@ -68,12 +68,12 @@ public class ExecutionPlanner { private final Config config; private final StreamManager streamManager; - private final StreamConfig streamConfig; + private final StreamConfig1 streamConfig; public ExecutionPlanner(Config config, StreamManager streamManager) { this.config = config; this.streamManager = streamManager; - this.streamConfig = new StreamConfig(config); + this.streamConfig = new StreamConfig1(config); } public ExecutionPlan plan(ApplicationDescriptorImpl appDesc) { diff --git a/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java b/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java index 7e94667c57..ce8feb8fe9 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java +++ b/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java @@ -39,7 +39,7 @@ import org.apache.samza.config.MapConfig; import org.apache.samza.config.SerializerConfig; import org.apache.samza.config.StorageConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.config.TaskConfig; import org.apache.samza.operators.KV; import org.apache.samza.operators.spec.JoinOperatorSpec; @@ -251,7 +251,7 @@ private void configureTables(Map generatedConfig, Config origina .map(sideInput -> StreamUtil.getSystemStreamFromNameOrId(originalConfig, sideInput)) .forEach(systemStream -> { inputs.add(StreamUtil.getNameFromSystemStream(systemStream)); - generatedConfig.put(String.format(StreamConfig.STREAM_PREFIX() + StreamConfig.BOOTSTRAP(), + generatedConfig.put(String.format(StreamConfig1.STREAM_PREFIX() + StreamConfig1.BOOTSTRAP(), systemStream.getSystem(), systemStream.getStream()), "true"); }); } @@ -314,14 +314,14 @@ private void configureSerdes(Map configs, Map { - String streamIdPrefix = String.format(StreamConfig.STREAM_ID_PREFIX(), streamId); - String keySerdeConfigKey = streamIdPrefix + StreamConfig.KEY_SERDE(); + String streamIdPrefix = String.format(StreamConfig1.STREAM_ID_PREFIX(), streamId); + String keySerdeConfigKey = streamIdPrefix + StreamConfig1.KEY_SERDE(); configs.put(keySerdeConfigKey, serdeUUIDs.get(serde)); }); streamMsgSerdes.forEach((streamId, serde) -> { - String streamIdPrefix = String.format(StreamConfig.STREAM_ID_PREFIX(), streamId); - String valueSerdeConfigKey = streamIdPrefix + StreamConfig.MSG_SERDE(); + String streamIdPrefix = String.format(StreamConfig1.STREAM_ID_PREFIX(), streamId); + String valueSerdeConfigKey = streamIdPrefix + StreamConfig1.MSG_SERDE(); configs.put(valueSerdeConfigKey, serdeUUIDs.get(serde)); }); diff --git a/samza-core/src/main/java/org/apache/samza/execution/JobPlanner.java b/samza-core/src/main/java/org/apache/samza/execution/JobPlanner.java index f30080456b..8c00040ab1 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/JobPlanner.java +++ b/samza-core/src/main/java/org/apache/samza/execution/JobPlanner.java @@ -35,7 +35,7 @@ import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; import org.apache.samza.config.ShellCommandConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.config.TaskConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,7 +126,7 @@ private Map getGeneratedConfig(String runId) { generatedConfig.put(ApplicationConfig.APP_RUN_ID, runId); } - StreamConfig streamConfig = new StreamConfig(userConfig); + StreamConfig1 streamConfig = new StreamConfig1(userConfig); Set inputStreamIds = new HashSet<>(appDesc.getInputStreamIds()); inputStreamIds.removeAll(appDesc.getOutputStreamIds()); // exclude intermediate streams ApplicationConfig.ApplicationMode mode = diff --git a/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java b/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java index 80d1bad626..50d6004ab3 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java +++ b/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java @@ -27,7 +27,7 @@ import org.apache.samza.config.ApplicationConfig; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.system.StreamSpec; import org.apache.samza.system.SystemStream; import org.apache.samza.util.StreamUtil; @@ -56,7 +56,7 @@ public class StreamEdge { this.isIntermediate = isIntermediate; // broadcast can be configured either by an operator or via the configs this.isBroadcast = - isBroadcast || new StreamConfig(config).getBroadcastEnabled(streamSpec.toSystemStream()); + isBroadcast || new StreamConfig1(config).getBroadcastEnabled(streamSpec.toSystemStream()); this.config = config; if (isBroadcast && isIntermediate) { partitions = 1; @@ -122,21 +122,21 @@ Config generateConfig() { Map streamConfig = new HashMap<>(); StreamSpec spec = getStreamSpec(); String streamId = spec.getId(); - streamConfig.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), streamId), spec.getSystemName()); - streamConfig.put(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), spec.getPhysicalName()); + streamConfig.put(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), streamId), spec.getSystemName()); + streamConfig.put(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), spec.getPhysicalName()); if (isIntermediate()) { - streamConfig.put(String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID(), streamId), "true"); - streamConfig.put(String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), streamId), "true"); + streamConfig.put(String.format(StreamConfig1.IS_INTERMEDIATE_FOR_STREAM_ID(), streamId), "true"); + streamConfig.put(String.format(StreamConfig1.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), streamId), "true"); // Setting offset.default to oldest only if the job is running in batch mode if (ApplicationConfig.ApplicationMode.BATCH.equals(new ApplicationConfig(config).getAppMode())) { - streamConfig.put(String.format(StreamConfig.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), streamId), "oldest"); + streamConfig.put(String.format(StreamConfig1.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), streamId), "oldest"); } - streamConfig.put(String.format(StreamConfig.PRIORITY_FOR_STREAM_ID(), streamId), String.valueOf(Integer.MAX_VALUE)); + streamConfig.put(String.format(StreamConfig1.PRIORITY_FOR_STREAM_ID(), streamId), String.valueOf(Integer.MAX_VALUE)); } spec.getConfig().forEach((property, value) -> { - streamConfig.put(String.format(StreamConfig.STREAM_ID_PREFIX(), streamId) + property, value); + streamConfig.put(String.format(StreamConfig1.STREAM_ID_PREFIX(), streamId) + property, value); }); return new MapConfig(streamConfig); diff --git a/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java b/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java index fe0d9fd0aa..aec1a467eb 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java +++ b/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java @@ -110,7 +110,7 @@ public void clearStreamsFromPreviousRun(Config prevConfig, ClassLoader classLoad ApplicationConfig appConfig = new ApplicationConfig(prevConfig); LOGGER.info("run.id from previous run is {}", appConfig.getRunId()); - StreamConfig streamConfig = new StreamConfig(prevConfig); + StreamConfig1 streamConfig = new StreamConfig1(prevConfig); //Find all intermediate streams and clean up Set intStreams = JavaConversions.asJavaCollection(streamConfig.getStreamIds()).stream() diff --git a/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java b/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java index 2b953211d8..f89b67f7bd 100644 --- a/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java +++ b/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java @@ -22,7 +22,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import org.apache.samza.config.Config; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.context.Context; import org.apache.samza.context.InternalTaskContext; import org.apache.samza.job.model.JobModel; @@ -99,7 +99,7 @@ public class OperatorImplGraph { */ public OperatorImplGraph(OperatorSpecGraph specGraph, Context context, Clock clock) { this.clock = clock; - StreamConfig streamConfig = new StreamConfig(context.getJobContext().getConfig()); + StreamConfig1 streamConfig = new StreamConfig1(context.getJobContext().getConfig()); this.internalTaskContext = new InternalTaskContext(context); Map producerTaskCounts = hasIntermediateStreams(specGraph) @@ -215,7 +215,7 @@ private OperatorImpl createAndRegisterOperatorImpl(OperatorSpec prevOperatorSpec */ OperatorImpl createOperatorImpl(OperatorSpec prevOperatorSpec, OperatorSpec operatorSpec, Context context) { Config config = context.getJobContext().getConfig(); - StreamConfig streamConfig = new StreamConfig(config); + StreamConfig1 streamConfig = new StreamConfig1(config); if (operatorSpec instanceof InputOperatorSpec) { return new InputOperatorImpl((InputOperatorSpec) operatorSpec); } else if (operatorSpec instanceof StreamOperatorSpec) { @@ -380,7 +380,7 @@ static Multimap getStreamToConsumerTasks(JobModel jobModel * @return mapping from output streams to input streams */ static Multimap getIntermediateToInputStreamsMap( - OperatorSpecGraph specGraph, StreamConfig streamConfig) { + OperatorSpecGraph specGraph, StreamConfig1 streamConfig) { Multimap outputToInputStreams = HashMultimap.create(); specGraph.getInputOperators().entrySet().stream() .forEach(entry -> { @@ -391,7 +391,7 @@ static Multimap getIntermediateToInputStreamsMap( } private static void computeOutputToInput(SystemStream input, OperatorSpec opSpec, - Multimap outputToInputStreams, StreamConfig streamConfig) { + Multimap outputToInputStreams, StreamConfig1 streamConfig) { if (opSpec instanceof PartitionByOperatorSpec) { PartitionByOperatorSpec spec = (PartitionByOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); diff --git a/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java b/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java index 293b9ab910..e60304d2a8 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java +++ b/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java @@ -57,8 +57,8 @@ public class StorageManagerUtil { /** * Fetch the starting offset for the input {@link SystemStreamPartition} * - * Note: The method doesn't respect {@link org.apache.samza.config.StreamConfig#CONSUMER_OFFSET_DEFAULT()} and - * {@link org.apache.samza.config.StreamConfig#CONSUMER_RESET_OFFSET()} configurations. It will use the locally + * Note: The method doesn't respect {@link org.apache.samza.config.StreamConfig1#CONSUMER_OFFSET_DEFAULT()} and + * {@link org.apache.samza.config.StreamConfig1#CONSUMER_RESET_OFFSET()} configurations. It will use the locally * checkpointed offset if it is valid, or fall back to oldest offset of the stream. * * @param ssp system stream partition for which starting offset is requested diff --git a/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java b/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java index 3de4f5d4a8..029d504c2d 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java +++ b/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java @@ -162,8 +162,8 @@ public StorageEngine getStore(String storeName) { /** * Gets the starting offset for the given side input {@link SystemStreamPartition}. * - * Note: The method doesn't respect {@link org.apache.samza.config.StreamConfig#CONSUMER_OFFSET_DEFAULT()} and - * {@link org.apache.samza.config.StreamConfig#CONSUMER_RESET_OFFSET()} configurations. It will use the local offset + * Note: The method doesn't respect {@link org.apache.samza.config.StreamConfig1#CONSUMER_OFFSET_DEFAULT()} and + * {@link org.apache.samza.config.StreamConfig1#CONSUMER_RESET_OFFSET()} configurations. It will use the local offset * file if it is valid, else it will fall back to oldest offset in the stream. * * @param ssp side input system stream partition to get the starting offset for diff --git a/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java b/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java index 96cdd6c4af..3f41406085 100644 --- a/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java +++ b/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java @@ -23,7 +23,7 @@ import java.util.stream.Collectors; import org.apache.samza.SamzaException; import org.apache.samza.config.Config; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.system.StreamSpec; import org.apache.samza.system.SystemStream; @@ -43,7 +43,7 @@ public static SystemStream getSystemStreamFromNameOrId(Config config, String str String.format("Invalid stream %s. Expected to be of the format streamId or systemName.streamName", stream)); } if (parts.length == 1) { - return new StreamConfig(config).streamIdToSystemStream(stream); + return new StreamConfig1(config).streamIdToSystemStream(stream); } else { return new SystemStream(parts[0], parts[1]); } @@ -78,11 +78,11 @@ public static String getNameFromSystemStream(SystemStream systemStream) { return systemStream.getSystem() + "." + systemStream.getStream(); } - public static Set getStreamSpecs(Set streamIds, StreamConfig streamConfig) { + public static Set getStreamSpecs(Set streamIds, StreamConfig1 streamConfig) { return streamIds.stream().map(streamId -> getStreamSpec(streamId, streamConfig)).collect(Collectors.toSet()); } - public static StreamSpec getStreamSpec(String streamId, StreamConfig streamConfig) { + public static StreamSpec getStreamSpec(String streamId, StreamConfig1 streamConfig) { String physicalName = streamConfig.getPhysicalName(streamId); String system = streamConfig.getSystem(streamId); Map streamProperties = streamConfig.getStreamProperties(streamId); diff --git a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala index cdeddb05fb..3f014d8f04 100644 --- a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala +++ b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala @@ -25,7 +25,7 @@ import java.util.concurrent.ConcurrentHashMap import org.apache.commons.lang3.StringUtils import org.apache.samza.SamzaException import org.apache.samza.annotation.InterfaceStability -import org.apache.samza.config.StreamConfig.Config2Stream +import org.apache.samza.config.StreamConfig1.Config2Stream import org.apache.samza.config.{Config, SystemConfig} import org.apache.samza.container.TaskName import org.apache.samza.startpoint.{Startpoint, StartpointManager} diff --git a/samza-core/src/main/scala/org/apache/samza/config/StreamConfig.scala b/samza-core/src/main/scala/org/apache/samza/config/StreamConfig1.scala similarity index 89% rename from samza-core/src/main/scala/org/apache/samza/config/StreamConfig.scala rename to samza-core/src/main/scala/org/apache/samza/config/StreamConfig1.scala index 6030b93786..6376c51db5 100644 --- a/samza-core/src/main/scala/org/apache/samza/config/StreamConfig.scala +++ b/samza-core/src/main/scala/org/apache/samza/config/StreamConfig1.scala @@ -24,7 +24,7 @@ import org.apache.samza.util.Logging import scala.collection.JavaConverters._ -object StreamConfig { +object StreamConfig1 { // Samza configs for streams val SAMZA_PROPERTY = "samza." val SYSTEM = SAMZA_PROPERTY + "system" @@ -55,7 +55,7 @@ object StreamConfig { val BOOTSTRAP_FOR_STREAM_ID = STREAM_ID_PREFIX + BOOTSTRAP val BROADCAST_FOR_STREAM_ID = STREAM_ID_PREFIX + BROADCAST - implicit def Config2Stream(config: Config) = new StreamConfig(config) + implicit def Config2Stream(config: Config) = new StreamConfig1(config) } /** @@ -75,13 +75,13 @@ object StreamConfig { * - samza property: property which is Samza-specific, which will have "samza." as a prefix (e.g. "samza.key.serde"); * this is in contrast to stream-specific properties which are related to specific stream technologies */ -class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { - def getStreamMsgSerde(systemStream: SystemStream) = nonEmptyOption(getSamzaProperty(systemStream, StreamConfig.MSG_SERDE)) +class StreamConfig1(config: Config) extends ScalaMapConfig(config) with Logging { + def getStreamMsgSerde(systemStream: SystemStream) = nonEmptyOption(getSamzaProperty(systemStream, StreamConfig1.MSG_SERDE)) - def getStreamKeySerde(systemStream: SystemStream) = nonEmptyOption(getSamzaProperty(systemStream, StreamConfig.KEY_SERDE)) + def getStreamKeySerde(systemStream: SystemStream) = nonEmptyOption(getSamzaProperty(systemStream, StreamConfig1.KEY_SERDE)) def getResetOffset(systemStream: SystemStream) = - Option(getSamzaProperty(systemStream, StreamConfig.CONSUMER_RESET_OFFSET)) match { + Option(getSamzaProperty(systemStream, StreamConfig1.CONSUMER_RESET_OFFSET)) match { case Some("true") => true case Some("false") => false case Some(resetOffset) => @@ -92,22 +92,22 @@ class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { } def isResetOffsetConfigured(systemStream: SystemStream) = - containsSamzaProperty(systemStream, StreamConfig.CONSUMER_RESET_OFFSET) + containsSamzaProperty(systemStream, StreamConfig1.CONSUMER_RESET_OFFSET) def getDefaultStreamOffset(systemStream: SystemStream) = - Option(getSamzaProperty(systemStream, StreamConfig.CONSUMER_OFFSET_DEFAULT)) + Option(getSamzaProperty(systemStream, StreamConfig1.CONSUMER_OFFSET_DEFAULT)) def isDefaultStreamOffsetConfigured(systemStream: SystemStream) = - containsSamzaProperty(systemStream, StreamConfig.CONSUMER_OFFSET_DEFAULT) + containsSamzaProperty(systemStream, StreamConfig1.CONSUMER_OFFSET_DEFAULT) def getBootstrapEnabled(systemStream: SystemStream) = - java.lang.Boolean.parseBoolean(getSamzaProperty(systemStream, StreamConfig.BOOTSTRAP)) + java.lang.Boolean.parseBoolean(getSamzaProperty(systemStream, StreamConfig1.BOOTSTRAP)) def getBroadcastEnabled(systemStream: SystemStream) = - java.lang.Boolean.parseBoolean(getSamzaProperty(systemStream, StreamConfig.BROADCAST)) + java.lang.Boolean.parseBoolean(getSamzaProperty(systemStream, StreamConfig1.BROADCAST)) def getPriority(systemStream: SystemStream) = - java.lang.Integer.parseInt(getSamzaProperty(systemStream, StreamConfig.PRIORITY, "-1")) + java.lang.Integer.parseInt(getSamzaProperty(systemStream, StreamConfig1.PRIORITY, "-1")) /** * Returns a list of all SystemStreams that have a serde defined from the config file. @@ -117,16 +117,16 @@ class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { val legacySystemStreams = subConf .asScala .keys - .filter(k => k.endsWith(StreamConfig.MSG_SERDE) || k.endsWith(StreamConfig.KEY_SERDE)) + .filter(k => k.endsWith(StreamConfig1.MSG_SERDE) || k.endsWith(StreamConfig1.KEY_SERDE)) .map(k => { val streamName = k.substring(0, k.length - 16 /* .samza.XXX.serde length */ ) new SystemStream(systemName, streamName) }).toSet - val systemStreams = subset(StreamConfig.STREAMS_PREFIX) + val systemStreams = subset(StreamConfig1.STREAMS_PREFIX) .asScala .keys - .filter(k => k.endsWith(StreamConfig.MSG_SERDE) || k.endsWith(StreamConfig.KEY_SERDE)) + .filter(k => k.endsWith(StreamConfig1.MSG_SERDE) || k.endsWith(StreamConfig1.KEY_SERDE)) .map(k => k.substring(0, k.length - 16 /* .samza.XXX.serde length */ )) .filter(streamId => systemName.equals(getSystem(streamId))) .map(streamId => streamIdToSystemStream(streamId)).toSet @@ -150,7 +150,7 @@ class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { */ def getStreamProperties(streamId: String) = { val allProperties = getAllStreamProperties(streamId) - val samzaProperties = allProperties.subset(StreamConfig.SAMZA_PROPERTY, false) + val samzaProperties = allProperties.subset(StreamConfig1.SAMZA_PROPERTY, false) val filteredStreamProperties: java.util.Map[String, String] = allProperties.asScala.filterKeys(k => !samzaProperties.containsKey(k)).asJava new MapConfig(filteredStreamProperties) @@ -168,7 +168,7 @@ class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { * @return the system name associated with the stream. */ def getSystem(streamId: String) = { - getOption(StreamConfig.SYSTEM_FOR_STREAM_ID format streamId) match { + getOption(StreamConfig1.SYSTEM_FOR_STREAM_ID format streamId) match { case Some(system) => system case _ => new JobConfig(config).getDefaultSystem().orElse(null) } @@ -182,7 +182,7 @@ class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { */ def getPhysicalName(streamId: String) = { // use streamId as the default physical name - get(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID format streamId, streamId) + get(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID format streamId, streamId) } /** @@ -191,7 +191,7 @@ class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { * @return true if the stream is intermediate */ def getIsIntermediateStream(streamId: String) = { - getBoolean(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID format streamId, false) + getBoolean(StreamConfig1.IS_INTERMEDIATE_FOR_STREAM_ID format streamId, false) } /** @@ -200,11 +200,11 @@ class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { * @return true if the committed messages of the stream can be deleted */ def getDeleteCommittedMessages(streamId: String) = { - getBoolean(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID format streamId, false) + getBoolean(StreamConfig1.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID format streamId, false) } def getIsBounded(streamId: String) = { - getBoolean(StreamConfig.IS_BOUNDED_FOR_STREAM_ID format streamId, false) + getBoolean(StreamConfig1.IS_BOUNDED_FOR_STREAM_ID format streamId, false) } /** @@ -213,7 +213,7 @@ class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { */ def getStreamIds(): Iterable[String] = { // StreamIds are not allowed to have '.' so the first index of '.' marks the end of the streamId. - subset(StreamConfig.STREAMS_PREFIX).asScala.keys.map(key => key.substring(0, key.indexOf("."))) + subset(StreamConfig1.STREAMS_PREFIX).asScala.keys.map(key => key.substring(0, key.indexOf("."))) } /** @@ -242,7 +242,7 @@ class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { * argument should have the value "samza.prop.key" */ private def getSamzaProperty(systemStream: SystemStream, property: String): String = { - if (!property.startsWith(StreamConfig.SAMZA_PROPERTY)) { + if (!property.startsWith(StreamConfig1.SAMZA_PROPERTY)) { throw new IllegalArgumentException( "Attempt to fetch a non samza property for SystemStream %s named %s" format(systemStream, property)) } @@ -285,7 +285,7 @@ class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { * argument should have the value "samza.prop.key" */ private def containsSamzaProperty(systemStream: SystemStream, property: String): Boolean = { - if (!property.startsWith(StreamConfig.SAMZA_PROPERTY)) { + if (!property.startsWith(StreamConfig1.SAMZA_PROPERTY)) { throw new IllegalArgumentException( "Attempt to fetch a non samza property for SystemStream %s named %s" format(systemStream, property)) } @@ -310,7 +310,7 @@ class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { } val systemConfig = new SystemConfig(config) val defaults = systemConfig.getDefaultStreamProperties(systemName) - val explicitConfigs = config.subset(StreamConfig.STREAM_PREFIX format(systemName, streamName), true) + val explicitConfigs = config.subset(StreamConfig1.STREAM_PREFIX format(systemName, streamName), true) new MapConfig(defaults, explicitConfigs) } @@ -328,7 +328,7 @@ class StreamConfig(config: Config) extends ScalaMapConfig(config) with Logging { * @return the merged map of config properties from both the legacy and new config styles */ private def getAllStreamProperties(streamId: String) = { - val allProperties = subset(StreamConfig.STREAM_ID_PREFIX format streamId) + val allProperties = subset(StreamConfig1.STREAM_ID_PREFIX format streamId) val inheritedLegacyProperties: java.util.Map[String, String] = getSystemStreamProperties(getSystem(streamId), getPhysicalName(streamId)) new MapConfig(java.util.Arrays.asList(inheritedLegacyProperties, allProperties)) diff --git a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala index f7c75f4d10..91498bcb53 100644 --- a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala +++ b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala @@ -31,7 +31,7 @@ import java.util.concurrent.{ExecutorService, Executors, ScheduledExecutorServic import com.google.common.annotations.VisibleForTesting import com.google.common.util.concurrent.ThreadFactoryBuilder import org.apache.samza.checkpoint.{CheckpointListener, OffsetManager, OffsetManagerMetrics} -import org.apache.samza.config.StreamConfig.Config2Stream +import org.apache.samza.config.StreamConfig1.Config2Stream import org.apache.samza.config._ import org.apache.samza.container.disk.DiskSpaceMonitor.Listener import org.apache.samza.container.disk.{DiskQuotaPolicyFactory, DiskSpaceMonitor, NoThrottlingDiskQuotaPolicyFactory, PollingScanDiskSpaceMonitor} diff --git a/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala b/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala index a17a790156..89b27f4520 100644 --- a/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala +++ b/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala @@ -25,7 +25,7 @@ import java.util.concurrent.ScheduledExecutorService import org.apache.samza.SamzaException import org.apache.samza.checkpoint.OffsetManager import org.apache.samza.config.Config -import org.apache.samza.config.StreamConfig.Config2Stream +import org.apache.samza.config.StreamConfig1.Config2Stream import org.apache.samza.context._ import org.apache.samza.job.model.{JobModel, TaskModel} import org.apache.samza.scheduler.{CallbackSchedulerImpl, EpochTimeScheduler, ScheduledCallback} diff --git a/samza-core/src/main/scala/org/apache/samza/job/JobRunner.scala b/samza-core/src/main/scala/org/apache/samza/job/JobRunner.scala index cbbca1a239..3c902694b6 100644 --- a/samza-core/src/main/scala/org/apache/samza/job/JobRunner.scala +++ b/samza-core/src/main/scala/org/apache/samza/job/JobRunner.scala @@ -124,7 +124,7 @@ class JobRunner(config: Config) extends Logging { val diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsSystemStreamName) val diagnosticsSysAdmin = systemAdmins.getSystemAdmin(diagnosticsSystemStream.getSystem) val diagnosticsStreamSpec = new StreamSpec(DIAGNOSTICS_STREAM_ID, diagnosticsSystemStream.getStream, - diagnosticsSystemStream.getSystem, new StreamConfig(config).getStreamProperties(DIAGNOSTICS_STREAM_ID)) + diagnosticsSystemStream.getSystem, new StreamConfig1(config).getStreamProperties(DIAGNOSTICS_STREAM_ID)) info("Creating diagnostics stream %s" format diagnosticsSystemStream.getStream) diagnosticsSysAdmin.start() diff --git a/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala b/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala index 27201c952a..deafe542b1 100644 --- a/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala +++ b/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala @@ -22,7 +22,7 @@ package org.apache.samza.metrics.reporter import org.apache.samza.util.{Logging, StreamUtil, Util} import org.apache.samza.SamzaException import org.apache.samza.config.{Config, JobConfig, MetricsConfig, SerializerConfig, SystemConfig} -import org.apache.samza.config.StreamConfig.Config2Stream +import org.apache.samza.config.StreamConfig1.Config2Stream import org.apache.samza.metrics.MetricsReporter import org.apache.samza.metrics.MetricsReporterFactory import org.apache.samza.metrics.MetricsRegistryMap diff --git a/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java b/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java index b91eeb0c19..23dc142910 100644 --- a/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java +++ b/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java @@ -47,211 +47,211 @@ public class TestStreamConfig { @Test public void testGetStreamMsgSerde() { String value = "my.msg.serde"; - doTestSamzaProperty(StreamConfig.MSG_SERDE(), value, + doTestSamzaProperty(StreamConfig1.MSG_SERDE(), value, (config, systemStream) -> assertEquals(Option.apply(value), config.getStreamMsgSerde(systemStream))); - doTestSamzaProperty(StreamConfig.MSG_SERDE(), "", + doTestSamzaProperty(StreamConfig1.MSG_SERDE(), "", (config, systemStream) -> assertEquals(Option.empty(), config.getStreamMsgSerde(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig.MSG_SERDE(), + doTestSamzaPropertyDoesNotExist(StreamConfig1.MSG_SERDE(), (config, systemStream) -> assertEquals(Option.empty(), config.getStreamMsgSerde(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig::getStreamMsgSerde); + doTestSamzaPropertyInvalidConfig(StreamConfig1::getStreamMsgSerde); } @Test public void testGetStreamKeySerde() { String value = "my.key.serde"; - doTestSamzaProperty(StreamConfig.KEY_SERDE(), value, + doTestSamzaProperty(StreamConfig1.KEY_SERDE(), value, (config, systemStream) -> assertEquals(Option.apply(value), config.getStreamKeySerde(systemStream))); - doTestSamzaProperty(StreamConfig.KEY_SERDE(), "", + doTestSamzaProperty(StreamConfig1.KEY_SERDE(), "", (config, systemStream) -> assertEquals(Option.empty(), config.getStreamKeySerde(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig.KEY_SERDE(), + doTestSamzaPropertyDoesNotExist(StreamConfig1.KEY_SERDE(), (config, systemStream) -> assertEquals(Option.empty(), config.getStreamKeySerde(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig::getStreamKeySerde); + doTestSamzaPropertyInvalidConfig(StreamConfig1::getStreamKeySerde); } @Test public void testGetResetOffset() { - doTestSamzaProperty(StreamConfig.CONSUMER_RESET_OFFSET(), "true", + doTestSamzaProperty(StreamConfig1.CONSUMER_RESET_OFFSET(), "true", (config, systemStream) -> assertTrue(config.getResetOffset(systemStream))); - doTestSamzaProperty(StreamConfig.CONSUMER_RESET_OFFSET(), "false", + doTestSamzaProperty(StreamConfig1.CONSUMER_RESET_OFFSET(), "false", (config, systemStream) -> assertFalse(config.getResetOffset(systemStream))); // if not true/false, then use false - doTestSamzaProperty(StreamConfig.CONSUMER_RESET_OFFSET(), "unknown_value", + doTestSamzaProperty(StreamConfig1.CONSUMER_RESET_OFFSET(), "unknown_value", (config, systemStream) -> assertFalse(config.getResetOffset(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig.CONSUMER_RESET_OFFSET(), + doTestSamzaPropertyDoesNotExist(StreamConfig1.CONSUMER_RESET_OFFSET(), (config, systemStream) -> assertFalse(config.getResetOffset(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig::getResetOffset); + doTestSamzaPropertyInvalidConfig(StreamConfig1::getResetOffset); } @Test public void testIsResetOffsetConfigured() { - doTestSamzaProperty(StreamConfig.CONSUMER_RESET_OFFSET(), "true", + doTestSamzaProperty(StreamConfig1.CONSUMER_RESET_OFFSET(), "true", (config, systemStream) -> assertTrue(config.isResetOffsetConfigured(systemStream))); - doTestSamzaProperty(StreamConfig.CONSUMER_RESET_OFFSET(), "false", + doTestSamzaProperty(StreamConfig1.CONSUMER_RESET_OFFSET(), "false", (config, systemStream) -> assertTrue(config.isResetOffsetConfigured(systemStream))); // if not true/false, then use false - doTestSamzaProperty(StreamConfig.CONSUMER_RESET_OFFSET(), "unknown_value", + doTestSamzaProperty(StreamConfig1.CONSUMER_RESET_OFFSET(), "unknown_value", (config, systemStream) -> assertTrue(config.isResetOffsetConfigured(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig.CONSUMER_RESET_OFFSET(), + doTestSamzaPropertyDoesNotExist(StreamConfig1.CONSUMER_RESET_OFFSET(), (config, systemStream) -> assertFalse(config.isResetOffsetConfigured(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig::isResetOffsetConfigured); + doTestSamzaPropertyInvalidConfig(StreamConfig1::isResetOffsetConfigured); } @Test public void testGetDefaultStreamOffset() { String value = "my_offset_default"; - doTestSamzaProperty(StreamConfig.CONSUMER_OFFSET_DEFAULT(), value, + doTestSamzaProperty(StreamConfig1.CONSUMER_OFFSET_DEFAULT(), value, (config, systemStream) -> assertEquals(Option.apply(value), config.getDefaultStreamOffset(systemStream))); - doTestSamzaProperty(StreamConfig.CONSUMER_OFFSET_DEFAULT(), "", + doTestSamzaProperty(StreamConfig1.CONSUMER_OFFSET_DEFAULT(), "", (config, systemStream) -> assertEquals(Option.apply(""), config.getDefaultStreamOffset(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig.CONSUMER_OFFSET_DEFAULT(), + doTestSamzaPropertyDoesNotExist(StreamConfig1.CONSUMER_OFFSET_DEFAULT(), (config, systemStream) -> assertEquals(Option.empty(), - new StreamConfig(config).getDefaultStreamOffset(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig::getDefaultStreamOffset); + new StreamConfig1(config).getDefaultStreamOffset(systemStream))); + doTestSamzaPropertyInvalidConfig(StreamConfig1::getDefaultStreamOffset); } @Test public void testIsDefaultStreamOffsetConfigured() { String value = "my_offset_default"; - doTestSamzaProperty(StreamConfig.CONSUMER_OFFSET_DEFAULT(), value, + doTestSamzaProperty(StreamConfig1.CONSUMER_OFFSET_DEFAULT(), value, (config, systemStream) -> assertTrue(config.isDefaultStreamOffsetConfigured(systemStream))); - doTestSamzaProperty(StreamConfig.CONSUMER_OFFSET_DEFAULT(), "", + doTestSamzaProperty(StreamConfig1.CONSUMER_OFFSET_DEFAULT(), "", (config, systemStream) -> assertTrue(config.isDefaultStreamOffsetConfigured(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig.CONSUMER_OFFSET_DEFAULT(), + doTestSamzaPropertyDoesNotExist(StreamConfig1.CONSUMER_OFFSET_DEFAULT(), (config, systemStream) -> assertFalse(config.isDefaultStreamOffsetConfigured(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig::isDefaultStreamOffsetConfigured); + doTestSamzaPropertyInvalidConfig(StreamConfig1::isDefaultStreamOffsetConfigured); } @Test public void testGetBootstrapEnabled() { - doTestSamzaProperty(StreamConfig.BOOTSTRAP(), "true", + doTestSamzaProperty(StreamConfig1.BOOTSTRAP(), "true", (config, systemStream) -> assertTrue(config.getBootstrapEnabled(systemStream))); - doTestSamzaProperty(StreamConfig.BOOTSTRAP(), "false", + doTestSamzaProperty(StreamConfig1.BOOTSTRAP(), "false", (config, systemStream) -> assertFalse(config.getBootstrapEnabled(systemStream))); // if not true/false, then use false - doTestSamzaProperty(StreamConfig.BOOTSTRAP(), "unknown_value", + doTestSamzaProperty(StreamConfig1.BOOTSTRAP(), "unknown_value", (config, systemStream) -> assertFalse(config.getBootstrapEnabled(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig.BOOTSTRAP(), + doTestSamzaPropertyDoesNotExist(StreamConfig1.BOOTSTRAP(), (config, systemStream) -> assertFalse(config.getBootstrapEnabled(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig::getBootstrapEnabled); + doTestSamzaPropertyInvalidConfig(StreamConfig1::getBootstrapEnabled); } @Test public void testGetBroadcastEnabled() { - doTestSamzaProperty(StreamConfig.BROADCAST(), "true", + doTestSamzaProperty(StreamConfig1.BROADCAST(), "true", (config, systemStream) -> assertTrue(config.getBroadcastEnabled(systemStream))); - doTestSamzaProperty(StreamConfig.BROADCAST(), "false", + doTestSamzaProperty(StreamConfig1.BROADCAST(), "false", (config, systemStream) -> assertFalse(config.getBroadcastEnabled(systemStream))); // if not true/false, then use false - doTestSamzaProperty(StreamConfig.BROADCAST(), "unknown_value", + doTestSamzaProperty(StreamConfig1.BROADCAST(), "unknown_value", (config, systemStream) -> assertFalse(config.getBroadcastEnabled(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig.BROADCAST(), + doTestSamzaPropertyDoesNotExist(StreamConfig1.BROADCAST(), (config, systemStream) -> assertFalse(config.getBroadcastEnabled(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig::getBroadcastEnabled); + doTestSamzaPropertyInvalidConfig(StreamConfig1::getBroadcastEnabled); } @Test public void testGetPriority() { - doTestSamzaProperty(StreamConfig.PRIORITY(), "0", + doTestSamzaProperty(StreamConfig1.PRIORITY(), "0", (config, systemStream) -> assertEquals(0, config.getPriority(systemStream))); - doTestSamzaProperty(StreamConfig.PRIORITY(), "100", + doTestSamzaProperty(StreamConfig1.PRIORITY(), "100", (config, systemStream) -> assertEquals(100, config.getPriority(systemStream))); - doTestSamzaProperty(StreamConfig.PRIORITY(), "-1", + doTestSamzaProperty(StreamConfig1.PRIORITY(), "-1", (config, systemStream) -> assertEquals(-1, config.getPriority(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig.PRIORITY(), + doTestSamzaPropertyDoesNotExist(StreamConfig1.PRIORITY(), (config, systemStream) -> assertEquals(-1, config.getPriority(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig::getPriority); + doTestSamzaPropertyInvalidConfig(StreamConfig1::getPriority); } @Test public void testGetSerdeStreams() { assertEquals(Collections.emptySet(), - JavaConverters.setAsJavaSetConverter(new StreamConfig(new MapConfig()).getSerdeStreams(SYSTEM)).asJava()); + JavaConverters.setAsJavaSetConverter(new StreamConfig1(new MapConfig()).getSerdeStreams(SYSTEM)).asJava()); // not key/msg serde property for "streams." - StreamConfig streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); + StreamConfig1 streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); assertEquals(Collections.emptySet(), JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); // not matching system for "streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig.KEY_SERDE(), UNUSED_VALUE, - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), "otherSystem"))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.KEY_SERDE(), UNUSED_VALUE, + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), "otherSystem"))); assertEquals(Collections.emptySet(), JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); // not key/msg serde property for "systems..streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); assertEquals(Collections.emptySet(), JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); // not matching system for "systems..streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), "otherSystem", STREAM_ID) + StreamConfig.KEY_SERDE(), + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), "otherSystem", STREAM_ID) + StreamConfig1.KEY_SERDE(), UNUSED_VALUE))); assertEquals(Collections.emptySet(), JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); // not matching system for "streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig.KEY_SERDE(), UNUSED_VALUE, - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), "otherSystem"))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.KEY_SERDE(), UNUSED_VALUE, + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), "otherSystem"))); assertEquals(Collections.emptySet(), JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); String serdeValue = "my.serde.class"; // key serde for "streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig.KEY_SERDE(), serdeValue))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.KEY_SERDE(), serdeValue))); assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); // msg serde for "streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig.MSG_SERDE(), serdeValue))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.MSG_SERDE(), serdeValue))); assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); // serde for "streams." with physical stream name mapping - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM, - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig.KEY_SERDE(), serdeValue))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM, + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.KEY_SERDE(), serdeValue))); assertEquals(Collections.singleton(new SystemStream(SYSTEM, PHYSICAL_STREAM)), JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); // key serde for "systems..streams." - streamConfig = new StreamConfig(new MapConfig( - ImmutableMap.of(String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + StreamConfig.KEY_SERDE(), + streamConfig = new StreamConfig1(new MapConfig( + ImmutableMap.of(String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + StreamConfig1.KEY_SERDE(), serdeValue))); assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); // msg serde for "systems..streams." - streamConfig = new StreamConfig(new MapConfig( - ImmutableMap.of(String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + StreamConfig.MSG_SERDE(), + streamConfig = new StreamConfig1(new MapConfig( + ImmutableMap.of(String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + StreamConfig1.MSG_SERDE(), serdeValue))); assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); // merge several different ways of providing serdes String streamIdWithPhysicalName = "streamIdWithPhysicalName"; - streamConfig = new StreamConfig(new MapConfig(new ImmutableMap.Builder() + streamConfig = new StreamConfig1(new MapConfig(new ImmutableMap.Builder() // need to map the stream ids to the system .put(JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM) // key and msg serde for "streams." - .put(String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig.KEY_SERDE(), serdeValue) - .put(String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig.MSG_SERDE(), serdeValue) + .put(String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.KEY_SERDE(), serdeValue) + .put(String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.MSG_SERDE(), serdeValue) // key serde for "streams." with physical stream name mapping - .put(String.format(StreamConfig.STREAM_ID_PREFIX(), streamIdWithPhysicalName) + StreamConfig.KEY_SERDE(), + .put(String.format(StreamConfig1.STREAM_ID_PREFIX(), streamIdWithPhysicalName) + StreamConfig1.KEY_SERDE(), serdeValue) - .put(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), streamIdWithPhysicalName), PHYSICAL_STREAM) + .put(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), streamIdWithPhysicalName), PHYSICAL_STREAM) // key serde for "systems..streams." - .put(String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, OTHER_STREAM_ID) + StreamConfig.KEY_SERDE(), + .put(String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, OTHER_STREAM_ID) + StreamConfig1.KEY_SERDE(), serdeValue) .build())); assertEquals(Sets.newHashSet(new SystemStream(SYSTEM, STREAM_ID), new SystemStream(SYSTEM, PHYSICAL_STREAM), @@ -261,44 +261,44 @@ public void testGetSerdeStreams() { @Test public void testGetStreamProperties() { - assertEquals(new MapConfig(), new StreamConfig(new MapConfig()).getStreamProperties(STREAM_ID)); + assertEquals(new MapConfig(), new StreamConfig1(new MapConfig()).getStreamProperties(STREAM_ID)); String propertyName = "stream.property.name"; // BEGIN: tests in which properties cannot be found in the config // not matching stream id for "streams." - StreamConfig streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), OTHER_STREAM_ID) + propertyName, UNUSED_VALUE))); + StreamConfig1 streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), OTHER_STREAM_ID) + propertyName, UNUSED_VALUE))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // not matching stream id for "systems..streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, OTHER_STREAM_ID) + propertyName, UNUSED_VALUE, - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), OTHER_STREAM_ID), SYSTEM))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, OTHER_STREAM_ID) + propertyName, UNUSED_VALUE, + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), OTHER_STREAM_ID), SYSTEM))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // no system mapping when using "systems..streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // ignore property with "samza" prefix for "streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // ignore property with "samza" prefix for "streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE, - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE, + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // should not map physical name back to stream id if physical name is passed as stream id - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(PHYSICAL_STREAM)); // BEGIN: tests in which properties can be found in the config @@ -306,65 +306,65 @@ public void testGetStreamProperties() { String propertyValue = "value"; // "streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, propertyValue))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, propertyValue))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // "systems..streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, propertyValue, - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, propertyValue, + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // "systems..default.stream." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, propertyValue, - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // use physical name mapping for "systems..streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, PHYSICAL_STREAM) + propertyName, propertyValue, + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, PHYSICAL_STREAM) + propertyName, propertyValue, // should not use stream id since there is physical stream - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // "streams." should override "systems..streams." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, propertyValue, + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, propertyValue, // should not use "systems..streams." since there is a "streams." config - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // "systems..streams." should override "systems..default.stream." - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, propertyValue, + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, propertyValue, // should not use "systems..default.stream." since there is a "systems..streams." String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, UNUSED_VALUE, - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // merge multiple ways of specifying configs - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( // "streams." - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + "from.stream.id.property", "fromStreamIdValue", + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + "from.stream.id.property", "fromStreamIdValue", // second "streams." property - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + "from.stream.id.other.property", + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + "from.stream.id.other.property", "fromStreamIdOtherValue", // "systems..streams." - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + "from.system.stream.property", + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + "from.system.stream.property", "fromSystemStreamValue", // need to map the stream id to a system - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of( "from.stream.id.property", "fromStreamIdValue", "from.stream.id.other.property", "fromStreamIdOtherValue", @@ -374,131 +374,131 @@ public void testGetStreamProperties() { @Test public void testGetSystem() { - assertNull(new StreamConfig(new MapConfig()).getSystem(STREAM_ID)); + assertNull(new StreamConfig1(new MapConfig()).getSystem(STREAM_ID)); // system is specified directly - StreamConfig streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + StreamConfig1 streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, JobConfig.JOB_DEFAULT_SYSTEM, "otherSystem", - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), OTHER_STREAM_ID), "otherSystem"))); + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), OTHER_STREAM_ID), "otherSystem"))); assertEquals(SYSTEM, streamConfig.getSystem(STREAM_ID)); // fall back to job default system - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), OTHER_STREAM_ID), "otherSystem"))); + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), OTHER_STREAM_ID), "otherSystem"))); assertEquals(SYSTEM, streamConfig.getSystem(STREAM_ID)); } @Test public void testGetPhysicalName() { - assertEquals(STREAM_ID, new StreamConfig(new MapConfig()).getPhysicalName(STREAM_ID)); + assertEquals(STREAM_ID, new StreamConfig1(new MapConfig()).getPhysicalName(STREAM_ID)); // ignore mapping for other stream ids - StreamConfig streamConfig = new StreamConfig(new MapConfig( - ImmutableMap.of(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), OTHER_STREAM_ID), PHYSICAL_STREAM))); + StreamConfig1 streamConfig = new StreamConfig1(new MapConfig( + ImmutableMap.of(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), OTHER_STREAM_ID), PHYSICAL_STREAM))); assertEquals(STREAM_ID, streamConfig.getPhysicalName(STREAM_ID)); - streamConfig = new StreamConfig(new MapConfig( - ImmutableMap.of(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfig1(new MapConfig( + ImmutableMap.of(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); assertEquals(PHYSICAL_STREAM, streamConfig.getPhysicalName(STREAM_ID)); } @Test public void testGetIsIntermediateStream() { - assertFalse(new StreamConfig(new MapConfig()).getIsIntermediateStream(STREAM_ID)); + assertFalse(new StreamConfig1(new MapConfig()).getIsIntermediateStream(STREAM_ID)); // ignore mapping for other stream ids - StreamConfig streamConfig = new StreamConfig(new MapConfig( - ImmutableMap.of(String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID(), OTHER_STREAM_ID), "true"))); + StreamConfig1 streamConfig = new StreamConfig1(new MapConfig( + ImmutableMap.of(String.format(StreamConfig1.IS_INTERMEDIATE_FOR_STREAM_ID(), OTHER_STREAM_ID), "true"))); assertFalse(streamConfig.getIsIntermediateStream(STREAM_ID)); // do not use stream id property if physical name is passed as input - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID(), STREAM_ID), "true", - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.IS_INTERMEDIATE_FOR_STREAM_ID(), STREAM_ID), "true", + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); assertFalse(streamConfig.getIsIntermediateStream(PHYSICAL_STREAM)); - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID(), STREAM_ID), "true"))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.IS_INTERMEDIATE_FOR_STREAM_ID(), STREAM_ID), "true"))); assertTrue(streamConfig.getIsIntermediateStream(STREAM_ID)); - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID(), STREAM_ID), "false"))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.IS_INTERMEDIATE_FOR_STREAM_ID(), STREAM_ID), "false"))); assertFalse(streamConfig.getIsIntermediateStream(STREAM_ID)); } @Test public void testGetDeleteCommittedMessages() { - assertFalse(new StreamConfig(new MapConfig()).getDeleteCommittedMessages(STREAM_ID)); + assertFalse(new StreamConfig1(new MapConfig()).getDeleteCommittedMessages(STREAM_ID)); // ignore mapping for other stream ids - StreamConfig streamConfig = new StreamConfig(new MapConfig( - ImmutableMap.of(String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), OTHER_STREAM_ID), + StreamConfig1 streamConfig = new StreamConfig1(new MapConfig( + ImmutableMap.of(String.format(StreamConfig1.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), OTHER_STREAM_ID), "true"))); assertFalse(streamConfig.getDeleteCommittedMessages(STREAM_ID)); // do not use stream id property if physical name is passed as input - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), STREAM_ID), "true", - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), STREAM_ID), "true", + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); assertFalse(streamConfig.getDeleteCommittedMessages(PHYSICAL_STREAM)); - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), STREAM_ID), "true"))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), STREAM_ID), "true"))); assertTrue(streamConfig.getDeleteCommittedMessages(STREAM_ID)); - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), STREAM_ID), "false"))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), STREAM_ID), "false"))); assertFalse(streamConfig.getDeleteCommittedMessages(STREAM_ID)); } @Test public void testGetIsBounded() { - assertFalse(new StreamConfig(new MapConfig()).getIsBounded(STREAM_ID)); + assertFalse(new StreamConfig1(new MapConfig()).getIsBounded(STREAM_ID)); // ignore mapping for other stream ids - StreamConfig streamConfig = new StreamConfig(new MapConfig( - ImmutableMap.of(String.format(StreamConfig.IS_BOUNDED_FOR_STREAM_ID(), OTHER_STREAM_ID), + StreamConfig1 streamConfig = new StreamConfig1(new MapConfig( + ImmutableMap.of(String.format(StreamConfig1.IS_BOUNDED_FOR_STREAM_ID(), OTHER_STREAM_ID), "true"))); assertFalse(streamConfig.getIsBounded(STREAM_ID)); // do not use stream id property if physical name is passed as input - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.IS_BOUNDED_FOR_STREAM_ID(), STREAM_ID), "true", - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.IS_BOUNDED_FOR_STREAM_ID(), STREAM_ID), "true", + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); assertFalse(streamConfig.getIsBounded(PHYSICAL_STREAM)); - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.IS_BOUNDED_FOR_STREAM_ID(), STREAM_ID), "true"))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.IS_BOUNDED_FOR_STREAM_ID(), STREAM_ID), "true"))); assertTrue(streamConfig.getIsBounded(STREAM_ID)); - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.IS_BOUNDED_FOR_STREAM_ID(), STREAM_ID), "false"))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.IS_BOUNDED_FOR_STREAM_ID(), STREAM_ID), "false"))); assertFalse(streamConfig.getIsBounded(STREAM_ID)); } @Test public void testGetStreamIds() { assertEquals(ImmutableList.of(), ImmutableList.copyOf( - JavaConverters.asJavaIterableConverter(new StreamConfig(new MapConfig()).getStreamIds()).asJava())); + JavaConverters.asJavaIterableConverter(new StreamConfig1(new MapConfig()).getStreamIds()).asJava())); - StreamConfig streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + ".property", "value"))); + StreamConfig1 streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + ".property", "value"))); assertEquals(ImmutableList.of(STREAM_ID), ImmutableList.copyOf(JavaConverters.asJavaIterableConverter(streamConfig.getStreamIds()).asJava())); - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + ".property.subProperty", "value"))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + ".property.subProperty", "value"))); assertEquals(ImmutableList.of(STREAM_ID), ImmutableList.copyOf(JavaConverters.asJavaIterableConverter(streamConfig.getStreamIds()).asJava())); - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + ".property0", "value", - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + ".property1", "value", - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + ".property.subProperty0", "value", - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + ".property.subProperty1", "value", - String.format(StreamConfig.STREAM_ID_PREFIX(), OTHER_STREAM_ID) + ".property", "value"))); + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + ".property0", "value", + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + ".property1", "value", + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + ".property.subProperty0", "value", + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + ".property.subProperty1", "value", + String.format(StreamConfig1.STREAM_ID_PREFIX(), OTHER_STREAM_ID) + ".property", "value"))); assertEquals(ImmutableList.of(STREAM_ID, OTHER_STREAM_ID), ImmutableList.copyOf(JavaConverters.asJavaIterableConverter(streamConfig.getStreamIds()).asJava())); } @@ -517,63 +517,63 @@ private static void doTestSamzaProperty(String propertyName, String propertyValu */ private static void doTestSamzaPropertyAccess(String propertyName, String value, SamzaPropertyAssertion assertion) { // streams.. - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, value, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM))), SYSTEM_STREAM); // systems..streams.. where stream id has no specified physical stream - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, // specify the system for the stream id - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))), + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))), SYSTEM_STREAM); // systems..streams.. where stream is the streamId, system is from job default system - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, // use job default system to get the system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM))), SYSTEM_STREAM); // systems..default.stream. where stream id has no specified physical stream - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // specify the system for the stream id - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))), + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))), SYSTEM_STREAM); // systems..streams.. where no system mapping (fall back to SystemStream) - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, // map the stream id to the physical stream - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); // systems..default.stream. where no system mapping (fall back to SystemStream) - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // map the stream id to the physical stream - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); // systems..streams.. where stream id has a physical name but property is from stream id - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, // use job default system to get the system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); // systems..default.stream. where stream id has a physical name but property is from stream id - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // use job default system to get the system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); } @@ -583,39 +583,39 @@ private static void doTestSamzaPropertyAccess(String propertyName, String value, private static void doTestSamzaPropertyAccessWithPhysicalStream(String propertyName, String value, SamzaPropertyAssertion assertion) { // streams.. - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, value, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); // systems..streams.. with a specific system for the stream id - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, PHYSICAL_STREAM) + propertyName, value, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, PHYSICAL_STREAM) + propertyName, value, // specify the system for the stream id - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); // systems..streams.. with system coming from job default system - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, PHYSICAL_STREAM) + propertyName, value, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, PHYSICAL_STREAM) + propertyName, value, // use job default system to get the system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); // systems..default.stream. - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // specify the system for the stream id - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); } @@ -626,19 +626,19 @@ private static void doTestSamzaPropertyAccessWithPhysicalStream(String propertyN */ private static void doTestSamzaPropertyPriority(String propertyName, String value, SamzaPropertyAssertion assertion) { // streams.. vs. systems..streams.. - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, value, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // this config should not be used - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE))), + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE))), SYSTEM_STREAM); // systems..streams.. vs. systems..default.stream. - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, // specify the system for the stream id - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, // this config should not be used String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, UNUSED_VALUE))), SYSTEM_STREAM); @@ -653,8 +653,8 @@ private static void doTestSamzaPropertyPriority(String propertyName, String valu * systems..streams.. without a system mapping in the config vs. * systems..default.stream. without a system mapping in the config */ - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, // this config should not be used String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, UNUSED_VALUE))), SYSTEM_STREAM); @@ -666,19 +666,19 @@ private static void doTestSamzaPropertyPriority(String propertyName, String valu */ private static void doTestSamzaPropertyMultipleStreams(String propertyName, String value, SamzaPropertyAssertion assertion) { // systems..default.stream. where stream id has no specified physical stream - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // specify the systems for the streams - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, - String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), OTHER_STREAM_ID), SYSTEM))), + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), OTHER_STREAM_ID), SYSTEM))), SYSTEM_STREAM); } private static void doTestSamzaPropertyInvalidConfig(SamzaPropertyLookup lookup) { // configure physical stream and have mapping from stream id to physical stream - StreamConfig streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM, - String.format(StreamConfig.STREAM_ID_PREFIX(), PHYSICAL_STREAM) + ".property", "value", + StreamConfig1 streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM, + String.format(StreamConfig1.STREAM_ID_PREFIX(), PHYSICAL_STREAM) + ".property", "value", JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM))); try { lookup.doLookup(streamConfig, SYSTEM_STREAM_PHYSICAL); @@ -688,9 +688,9 @@ private static void doTestSamzaPropertyInvalidConfig(SamzaPropertyLookup lookup) } // two separate stream ids map to same physical stream - streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM, - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), OTHER_STREAM_ID), PHYSICAL_STREAM, + streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM, + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), OTHER_STREAM_ID), PHYSICAL_STREAM, JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM))); try { lookup.doLookup(streamConfig, SYSTEM_STREAM_PHYSICAL); @@ -701,42 +701,42 @@ private static void doTestSamzaPropertyInvalidConfig(SamzaPropertyLookup lookup) } private static void doTestSamzaPropertyDoesNotExist(String propertyName, SamzaPropertyAssertion assertion) { - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( // just put in some value which will be ignored - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))), + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))), SYSTEM_STREAM); /* * Won't use streams.. if streamId is mapped to a physical stream */ - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, UNUSED_VALUE, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, UNUSED_VALUE, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); /* * Won't use systems..streams.. if stream is mapped to a physical stream and the property * is only specified using the physical stream */ - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, PHYSICAL_STREAM) + propertyName, UNUSED_VALUE, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, PHYSICAL_STREAM) + propertyName, UNUSED_VALUE, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); /* * Won't use systems..streams.. if stream is mapped to a physical stream and there is no * system mapping for the stream id */ - assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( - String.format(StreamConfig.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, + assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, // map the stream id to the physical stream - String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); } @@ -745,7 +745,7 @@ private static void doTestSamzaPropertyDoesNotExist(String propertyName, SamzaPr */ @FunctionalInterface private interface SamzaPropertyAssertion { - void doAssertion(StreamConfig streamConfig, SystemStream systemStream); + void doAssertion(StreamConfig1 streamConfig, SystemStream systemStream); } /** @@ -753,6 +753,6 @@ private interface SamzaPropertyAssertion { */ @FunctionalInterface private interface SamzaPropertyLookup { - void doLookup(StreamConfig streamConfig, SystemStream systemStream); + void doLookup(StreamConfig1 streamConfig, SystemStream systemStream); } } \ No newline at end of file diff --git a/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java b/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java index 5512bbcaa8..c1d6223342 100644 --- a/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java +++ b/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java @@ -157,13 +157,13 @@ public void testGetSystemKeySerde() { // value specified explicitly Config config = - new MapConfig(ImmutableMap.of(defaultStreamPrefixSystem1 + StreamConfig.KEY_SERDE(), system1KeySerde)); + new MapConfig(ImmutableMap.of(defaultStreamPrefixSystem1 + StreamConfig1.KEY_SERDE(), system1KeySerde)); SystemConfig systemConfig = new SystemConfig(config); assertEquals(system1KeySerde, systemConfig.getSystemKeySerde(MOCK_SYSTEM_NAME1).get()); // default stream property is unspecified, try fall back config key config = new MapConfig( - ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig.KEY_SERDE(), + ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig1.KEY_SERDE(), system1KeySerde)); systemConfig = new SystemConfig(config); assertEquals(system1KeySerde, systemConfig.getSystemKeySerde(MOCK_SYSTEM_NAME1).get()); @@ -171,15 +171,15 @@ public void testGetSystemKeySerde() { // default stream property is empty string, try fall back config key config = new MapConfig(ImmutableMap.of( // default stream property is empty - defaultStreamPrefixSystem1 + StreamConfig.KEY_SERDE(), "", + defaultStreamPrefixSystem1 + StreamConfig1.KEY_SERDE(), "", // fall back entry - String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig.KEY_SERDE(), system1KeySerde)); + String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig1.KEY_SERDE(), system1KeySerde)); systemConfig = new SystemConfig(config); assertEquals(system1KeySerde, systemConfig.getSystemKeySerde(MOCK_SYSTEM_NAME1).get()); // default stream property is unspecified, fall back is also empty config = new MapConfig( - ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig.KEY_SERDE(), + ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig1.KEY_SERDE(), "")); systemConfig = new SystemConfig(config); assertFalse(systemConfig.getSystemKeySerde(MOCK_SYSTEM_NAME1).isPresent()); @@ -197,13 +197,13 @@ public void testGetSystemMsgSerde() { // value specified explicitly Config config = - new MapConfig(ImmutableMap.of(defaultStreamPrefixSystem1 + StreamConfig.MSG_SERDE(), system1MsgSerde)); + new MapConfig(ImmutableMap.of(defaultStreamPrefixSystem1 + StreamConfig1.MSG_SERDE(), system1MsgSerde)); SystemConfig systemConfig = new SystemConfig(config); assertEquals(system1MsgSerde, systemConfig.getSystemMsgSerde(MOCK_SYSTEM_NAME1).get()); // default stream property is unspecified, try fall back config msg config = new MapConfig( - ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig.MSG_SERDE(), + ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig1.MSG_SERDE(), system1MsgSerde)); systemConfig = new SystemConfig(config); assertEquals(system1MsgSerde, systemConfig.getSystemMsgSerde(MOCK_SYSTEM_NAME1).get()); @@ -211,15 +211,15 @@ public void testGetSystemMsgSerde() { // default stream property is empty string, try fall back config msg config = new MapConfig(ImmutableMap.of( // default stream property is empty - defaultStreamPrefixSystem1 + StreamConfig.MSG_SERDE(), "", + defaultStreamPrefixSystem1 + StreamConfig1.MSG_SERDE(), "", // fall back entry - String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig.MSG_SERDE(), system1MsgSerde)); + String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig1.MSG_SERDE(), system1MsgSerde)); systemConfig = new SystemConfig(config); assertEquals(system1MsgSerde, systemConfig.getSystemMsgSerde(MOCK_SYSTEM_NAME1).get()); // default stream property is unspecified, fall back is also empty config = new MapConfig( - ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig.MSG_SERDE(), + ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig1.MSG_SERDE(), "")); systemConfig = new SystemConfig(config); assertFalse(systemConfig.getSystemMsgSerde(MOCK_SYSTEM_NAME1).isPresent()); diff --git a/samza-core/src/test/java/org/apache/samza/execution/TestExecutionPlanner.java b/samza-core/src/test/java/org/apache/samza/execution/TestExecutionPlanner.java index 6a65286aa2..be7b15b20c 100644 --- a/samza-core/src/test/java/org/apache/samza/execution/TestExecutionPlanner.java +++ b/samza-core/src/test/java/org/apache/samza/execution/TestExecutionPlanner.java @@ -40,7 +40,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig$; +import org.apache.samza.config.StreamConfig1$; import org.apache.samza.config.TaskConfig; import org.apache.samza.serializers.StringSerde; import org.apache.samza.system.descriptors.GenericInputDescriptor; @@ -640,7 +640,7 @@ public void testDefaultPartitions() { @Test public void testBroadcastConfig() { Map map = new HashMap<>(config); - map.put(String.format(StreamConfig$.MODULE$.BROADCAST_FOR_STREAM_ID(), "input1"), "true"); + map.put(String.format(StreamConfig1$.MODULE$.BROADCAST_FOR_STREAM_ID(), "input1"), "true"); Config cfg = new MapConfig(map); ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager); diff --git a/samza-core/src/test/java/org/apache/samza/execution/TestStreamEdge.java b/samza-core/src/test/java/org/apache/samza/execution/TestStreamEdge.java index 94275dd660..1be25862d4 100644 --- a/samza-core/src/test/java/org/apache/samza/execution/TestStreamEdge.java +++ b/samza-core/src/test/java/org/apache/samza/execution/TestStreamEdge.java @@ -22,7 +22,7 @@ import org.apache.samza.config.ApplicationConfig; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.system.StreamSpec; import org.junit.Test; @@ -67,7 +67,7 @@ public void testGenerateConfig() { StreamSpec spec = new StreamSpec("stream-1", "physical-stream-1", "system-1", Collections.singletonMap("property1", "haha")); StreamEdge edge = new StreamEdge(spec, false, false, new MapConfig()); Config config = edge.generateConfig(); - StreamConfig streamConfig = new StreamConfig(config); + StreamConfig1 streamConfig = new StreamConfig1(config); assertEquals(streamConfig.getSystem(spec.getId()), "system-1"); assertEquals(streamConfig.getPhysicalName(spec.getId()), "physical-stream-1"); assertEquals(streamConfig.getIsIntermediateStream(spec.getId()), false); @@ -77,12 +77,12 @@ public void testGenerateConfig() { spec = new StreamSpec("stream-1", "physical-stream-1", "system-1", Collections.singletonMap("property1", "haha")); edge = new StreamEdge(spec, false, false, new MapConfig()); config = edge.generateConfig(); - streamConfig = new StreamConfig(config); + streamConfig = new StreamConfig1(config); // intermediate stream edge = new StreamEdge(spec, true, false, new MapConfig()); config = edge.generateConfig(); - streamConfig = new StreamConfig(config); + streamConfig = new StreamConfig1(config); assertEquals(streamConfig.getIsIntermediateStream(spec.getId()), true); assertEquals(streamConfig.getPriority(spec.toSystemStream()), Integer.MAX_VALUE); } diff --git a/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImplGraph.java b/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImplGraph.java index 579c028831..b908a3c8f6 100644 --- a/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImplGraph.java +++ b/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImplGraph.java @@ -26,7 +26,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.container.TaskName; import org.apache.samza.context.Context; import org.apache.samza.context.MockContext; @@ -504,7 +504,7 @@ public void testGetOutputToInputStreams() { }, config); Multimap outputToInput = - OperatorImplGraph.getIntermediateToInputStreamsMap(graphSpec.getOperatorSpecGraph(), new StreamConfig(config)); + OperatorImplGraph.getIntermediateToInputStreamsMap(graphSpec.getOperatorSpecGraph(), new StreamConfig1(config)); Collection inputs = outputToInput.get(new SystemStream(intSystem, intStreamId2)); assertEquals(inputs.size(), 2); assertTrue(inputs.contains(new SystemStream(inputSystem, inputStreamId1))); diff --git a/samza-core/src/test/java/org/apache/samza/testUtils/StreamTestUtils.java b/samza-core/src/test/java/org/apache/samza/testUtils/StreamTestUtils.java index 0aebd30aa2..fe710938d4 100644 --- a/samza-core/src/test/java/org/apache/samza/testUtils/StreamTestUtils.java +++ b/samza-core/src/test/java/org/apache/samza/testUtils/StreamTestUtils.java @@ -19,7 +19,7 @@ package org.apache.samza.testUtils; import java.util.Map; -import org.apache.samza.config.StreamConfig$; +import org.apache.samza.config.StreamConfig1$; public class StreamTestUtils { @@ -33,7 +33,7 @@ public class StreamTestUtils { */ public static void addStreamConfigs(Map configs, String streamId, String systemName, String physicalName) { - configs.put(String.format(StreamConfig$.MODULE$.SYSTEM_FOR_STREAM_ID(), streamId), systemName); - configs.put(String.format(StreamConfig$.MODULE$.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), physicalName); + configs.put(String.format(StreamConfig1$.MODULE$.SYSTEM_FOR_STREAM_ID(), streamId), systemName); + configs.put(String.format(StreamConfig1$.MODULE$.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), physicalName); } } \ No newline at end of file diff --git a/samza-core/src/test/java/org/apache/samza/util/TestStreamUtil.java b/samza-core/src/test/java/org/apache/samza/util/TestStreamUtil.java index 341eb24818..4d3d876fb9 100644 --- a/samza-core/src/test/java/org/apache/samza/util/TestStreamUtil.java +++ b/samza-core/src/test/java/org/apache/samza/util/TestStreamUtil.java @@ -23,7 +23,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.system.StreamSpec; import org.junit.Test; @@ -49,10 +49,10 @@ public class TestStreamUtil { @Test public void testGetStreamWithPhysicalNameInConfig() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig.SYSTEM(), TEST_SYSTEM); + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, + StreamConfig1.SYSTEM(), TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); assertEquals(TEST_PHYSICAL_NAME, spec.getPhysicalName()); } @@ -62,9 +62,9 @@ public void testGetStreamWithPhysicalNameInConfig() { @Test public void testGetStreamWithoutPhysicalNameInConfig() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.SYSTEM(), TEST_SYSTEM); + StreamConfig1.SYSTEM(), TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); assertEquals(STREAM_ID, spec.getPhysicalName()); } @@ -73,10 +73,10 @@ public void testGetStreamWithoutPhysicalNameInConfig() { @Test public void testGetStreamWithSystemAtStreamScopeInConfig() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig.SYSTEM(), TEST_SYSTEM); + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, + StreamConfig1.SYSTEM(), TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); assertEquals(TEST_SYSTEM, spec.getSystemName()); } @@ -85,10 +85,10 @@ public void testGetStreamWithSystemAtStreamScopeInConfig() { @Test public void testGetStreamWithSystemAtDefaultScopeInConfig() { Config config = addConfigs(buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME), + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME), JobConfig.JOB_DEFAULT_SYSTEM, TEST_DEFAULT_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); assertEquals(TEST_DEFAULT_SYSTEM, spec.getSystemName()); } @@ -97,11 +97,11 @@ public void testGetStreamWithSystemAtDefaultScopeInConfig() { @Test public void testGetStreamWithSystemAtBothScopesInConfig() { Config config = addConfigs(buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig.SYSTEM(), TEST_SYSTEM), + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, + StreamConfig1.SYSTEM(), TEST_SYSTEM), JobConfig.JOB_DEFAULT_SYSTEM, TEST_DEFAULT_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); assertEquals(TEST_SYSTEM, spec.getSystemName()); } @@ -110,9 +110,9 @@ public void testGetStreamWithSystemAtBothScopesInConfig() { @Test(expected = IllegalArgumentException.class) public void testGetStreamWithOutSystemInConfig() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME); + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); assertEquals(TEST_SYSTEM, spec.getSystemName()); } @@ -121,13 +121,13 @@ public void testGetStreamWithOutSystemInConfig() { @Test public void testGetStreamPropertiesPassthrough() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig.SYSTEM(), TEST_SYSTEM, + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, + StreamConfig1.SYSTEM(), TEST_SYSTEM, "systemProperty1", "systemValue1", "systemProperty2", "systemValue2", "systemProperty3", "systemValue3"); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); Map properties = spec.getConfig(); assertEquals(3, properties.size()); @@ -143,35 +143,35 @@ public void testGetStreamPropertiesPassthrough() { @Test public void testGetStreamSamzaPropertiesOmitted() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig.SYSTEM(), TEST_SYSTEM, + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, + StreamConfig1.SYSTEM(), TEST_SYSTEM, "systemProperty1", "systemValue1", "systemProperty2", "systemValue2", "systemProperty3", "systemValue3"); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); Map properties = spec.getConfig(); assertEquals(3, properties.size()); - assertNull(properties.get(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID))); - assertNull(properties.get(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID))); - assertNull(spec.get(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID))); - assertNull(spec.get(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID))); + assertNull(properties.get(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID))); + assertNull(properties.get(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID))); + assertNull(spec.get(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID))); + assertNull(spec.get(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID))); } @Test public void testStreamConfigOverrides() { final String sysStreamPrefix = String.format("systems.%s.streams.%s.", TEST_SYSTEM, TEST_PHYSICAL_NAME); Config config = addConfigs(buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig.SYSTEM(), TEST_SYSTEM, + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, + StreamConfig1.SYSTEM(), TEST_SYSTEM, "systemProperty1", "systemValue1", "systemProperty2", "systemValue2", "systemProperty3", "systemValue3"), sysStreamPrefix + "systemProperty4", "systemValue4", sysStreamPrefix + "systemProperty2", "systemValue8"); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); Map properties = spec.getConfig(); assertEquals(4, properties.size()); @@ -183,14 +183,14 @@ public void testStreamConfigOverrides() { @Test public void testStreamConfigOverridesWithSystemDefaults() { Config config = addConfigs(buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig.SYSTEM(), TEST_SYSTEM, + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, + StreamConfig1.SYSTEM(), TEST_SYSTEM, "segment.bytes", "5309"), String.format("systems.%s.default.stream.replication.factor", TEST_SYSTEM), "4", // System default property String.format("systems.%s.default.stream.segment.bytest", TEST_SYSTEM), "867" ); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); Map properties = spec.getConfig(); assertEquals(3, properties.size()); @@ -202,10 +202,10 @@ public void testStreamConfigOverridesWithSystemDefaults() { @Test public void testGetStreamPhysicalNameArgSimple() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME2, // This should be ignored because of the explicit arg - StreamConfig.SYSTEM(), TEST_SYSTEM); + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME2, // This should be ignored because of the explicit arg + StreamConfig1.SYSTEM(), TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); assertEquals(STREAM_ID, spec.getId()); assertEquals(TEST_PHYSICAL_NAME2, spec.getPhysicalName()); @@ -216,10 +216,10 @@ public void testGetStreamPhysicalNameArgSimple() { @Test public void testGetStreamPhysicalNameArgSpecialCharacters() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME_SPECIAL_CHARS, - StreamConfig.SYSTEM(), TEST_SYSTEM); + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME_SPECIAL_CHARS, + StreamConfig1.SYSTEM(), TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); assertEquals(TEST_PHYSICAL_NAME_SPECIAL_CHARS, spec.getPhysicalName()); } @@ -227,10 +227,10 @@ public void testGetStreamPhysicalNameArgSpecialCharacters() { @Test public void testGetStreamPhysicalNameArgNull() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), null, - StreamConfig.SYSTEM(), TEST_SYSTEM); + StreamConfig1.PHYSICAL_NAME(), null, + StreamConfig1.SYSTEM(), TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); assertNull(spec.getPhysicalName()); } @@ -238,10 +238,10 @@ public void testGetStreamPhysicalNameArgNull() { @Test public void testGetStreamSystemNameArgValid() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, // This should be ignored because of the explicit arg - StreamConfig.SYSTEM(), TEST_SYSTEM); // This too + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, // This should be ignored because of the explicit arg + StreamConfig1.SYSTEM(), TEST_SYSTEM); // This too - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); assertEquals(STREAM_ID, spec.getId()); assertEquals(TEST_PHYSICAL_NAME, spec.getPhysicalName()); @@ -252,57 +252,57 @@ public void testGetStreamSystemNameArgValid() { @Test(expected = IllegalArgumentException.class) public void testGetStreamSystemNameArgInvalid() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig.SYSTEM(), TEST_SYSTEM_INVALID); + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, + StreamConfig1.SYSTEM(), TEST_SYSTEM_INVALID); - StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); } // Empty strings are NOT allowed for system name, because it's used as an identifier in the config. @Test(expected = IllegalArgumentException.class) public void testGetStreamSystemNameArgEmpty() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig.SYSTEM(), ""); + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, + StreamConfig1.SYSTEM(), ""); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); } // Null is not allowed IllegalArgumentException system name. @Test(expected = IllegalArgumentException.class) public void testGetStreamSystemNameArgNull() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig.SYSTEM(), null); + StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, + StreamConfig1.SYSTEM(), null); - StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); + StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); } // Special characters are NOT allowed for streamId, because it's used as an identifier in the config. @Test(expected = IllegalArgumentException.class) public void testGetStreamStreamIdInvalid() { Config config = buildStreamConfig(STREAM_ID_INVALID, - StreamConfig.SYSTEM(), TEST_SYSTEM); + StreamConfig1.SYSTEM(), TEST_SYSTEM); - StreamUtil.getStreamSpec(STREAM_ID_INVALID, new StreamConfig(config)); + StreamUtil.getStreamSpec(STREAM_ID_INVALID, new StreamConfig1(config)); } // Empty strings are NOT allowed for streamId, because it's used as an identifier in the config. @Test(expected = IllegalArgumentException.class) public void testGetStreamStreamIdEmpty() { Config config = buildStreamConfig("", - StreamConfig.SYSTEM(), TEST_SYSTEM); + StreamConfig1.SYSTEM(), TEST_SYSTEM); - StreamUtil.getStreamSpec("", new StreamConfig(config)); + StreamUtil.getStreamSpec("", new StreamConfig1(config)); } // Null is not allowed for streamId. @Test(expected = IllegalArgumentException.class) public void testGetStreamStreamIdNull() { Config config = buildStreamConfig(null, - StreamConfig.SYSTEM(), TEST_SYSTEM); + StreamConfig1.SYSTEM(), TEST_SYSTEM); - StreamUtil.getStreamSpec(null, new StreamConfig(config)); + StreamUtil.getStreamSpec(null, new StreamConfig1(config)); } @@ -311,7 +311,7 @@ public void testGetStreamStreamIdNull() { private Config buildStreamConfig(String streamId, String... kvs) { // inject streams.x. into each key for (int i = 0; i < kvs.length - 1; i += 2) { - kvs[i] = String.format(StreamConfig.STREAM_ID_PREFIX(), streamId) + kvs[i]; + kvs[i] = String.format(StreamConfig1.STREAM_ID_PREFIX(), streamId) + kvs[i]; } return buildConfig(kvs); } diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index 1986bea54f..1dceff875b 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -57,7 +57,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.KafkaConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.config.SystemConfig; import org.apache.samza.startpoint.Startpoint; import org.apache.samza.startpoint.StartpointOldest; @@ -687,7 +687,7 @@ static Map getIntermediateStreamProperties(Config config) { ApplicationConfig appConfig = new ApplicationConfig(config); if (appConfig.getAppMode() == ApplicationConfig.ApplicationMode.BATCH) { - StreamConfig streamConfig = new StreamConfig(config); + StreamConfig1 streamConfig = new StreamConfig1(config); intermedidateStreamProperties = JavaConverters.asJavaCollectionConverter(streamConfig.getStreamIds()) .asJavaCollection() .stream() diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala index deb80569ec..76df02e730 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala @@ -108,7 +108,7 @@ class KafkaSystemFactory extends SystemFactory with Logging { def getIntermediateStreamProperties(config: Config): Map[String, Properties] = { val appConfig = new ApplicationConfig(config) if (appConfig.getAppMode == ApplicationMode.BATCH) { - val streamConfig = new StreamConfig(config) + val streamConfig = new StreamConfig1(config) streamConfig.getStreamIds().filter(streamConfig.getIsIntermediateStream(_)).map(streamId => { // only the override here val properties = new Properties() diff --git a/samza-log4j/src/main/java/org/apache/samza/config/Log4jSystemConfig.java b/samza-log4j/src/main/java/org/apache/samza/config/Log4jSystemConfig.java index b449f90801..a7a334b50e 100644 --- a/samza-log4j/src/main/java/org/apache/samza/config/Log4jSystemConfig.java +++ b/samza-log4j/src/main/java/org/apache/samza/config/Log4jSystemConfig.java @@ -81,7 +81,7 @@ public String getSerdeClass(String name) { } public String getStreamSerdeName(String systemName, String streamName) { - StreamConfig streamConfig = new StreamConfig(this); + StreamConfig1 streamConfig = new StreamConfig1(this); scala.Option option = streamConfig.getStreamMsgSerde(new SystemStream(systemName, streamName)); return option.isEmpty() ? null : option.get(); } diff --git a/samza-log4j2/src/main/java/org/apache/samza/config/Log4jSystemConfig.java b/samza-log4j2/src/main/java/org/apache/samza/config/Log4jSystemConfig.java index b449f90801..a7a334b50e 100644 --- a/samza-log4j2/src/main/java/org/apache/samza/config/Log4jSystemConfig.java +++ b/samza-log4j2/src/main/java/org/apache/samza/config/Log4jSystemConfig.java @@ -81,7 +81,7 @@ public String getSerdeClass(String name) { } public String getStreamSerdeName(String systemName, String streamName) { - StreamConfig streamConfig = new StreamConfig(this); + StreamConfig1 streamConfig = new StreamConfig1(this); scala.Option option = streamConfig.getStreamMsgSerde(new SystemStream(systemName, streamName)); return option.isEmpty() ? null : option.get(); } diff --git a/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java b/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java index d92faaef96..51775fbfb5 100644 --- a/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java +++ b/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java @@ -28,7 +28,7 @@ import org.apache.commons.lang.Validate; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.table.descriptors.CachingTableDescriptor; import org.apache.samza.table.descriptors.RemoteTableDescriptor; import org.apache.samza.table.descriptors.TableDescriptor; @@ -100,11 +100,11 @@ public SqlIOConfig(String systemName, String streamName, List sourcePart if (!isRemoteTable()) { // The below config is required for local table and streams but not for remote table. - streamConfigs.put(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), streamName); + streamConfigs.put(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), streamName); if (tableDescriptor != null) { // For local table, set the bootstrap config and default offset to oldest - streamConfigs.put(String.format(StreamConfig.BOOTSTRAP_FOR_STREAM_ID(), streamId), "true"); - streamConfigs.put(String.format(StreamConfig.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), streamId), "oldest"); + streamConfigs.put(String.format(StreamConfig1.BOOTSTRAP_FOR_STREAM_ID(), streamId), "true"); + streamConfigs.put(String.format(StreamConfig1.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), streamId), "oldest"); } } diff --git a/samza-sql/src/test/java/org/apache/samza/sql/translator/TestQueryTranslator.java b/samza-sql/src/test/java/org/apache/samza/sql/translator/TestQueryTranslator.java index efe38961c4..4819d8b588 100644 --- a/samza-sql/src/test/java/org/apache/samza/sql/translator/TestQueryTranslator.java +++ b/samza-sql/src/test/java/org/apache/samza/sql/translator/TestQueryTranslator.java @@ -29,7 +29,7 @@ import org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.context.ContainerContext; import org.apache.samza.context.Context; import org.apache.samza.operators.OperatorSpecGraph; @@ -83,7 +83,7 @@ public void testTranslate() { translator.translate(queryInfo.get(0), appDesc, 0); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); - StreamConfig streamConfig = new StreamConfig(samzaConfig); + StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); @@ -122,7 +122,7 @@ public void testTranslateFanIn() { translator.translate(queryInfo.get(1), appDesc, 1); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); - StreamConfig streamConfig = new StreamConfig(samzaConfig); + StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); String inputStreamId1 = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem1 = streamConfig.getSystem(inputStreamId1); String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1); @@ -167,7 +167,7 @@ public void testTranslateFanOut() { translator.translate(queryInfo.get(1), appDesc, 1); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); - StreamConfig streamConfig = new StreamConfig(samzaConfig); + StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); @@ -211,7 +211,7 @@ public void testTranslateMultiSql() { translator.translate(queryInfo.get(1), appDesc, 1); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); - StreamConfig streamConfig = new StreamConfig(samzaConfig); + StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); String inputStreamId1 = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem1 = streamConfig.getSystem(inputStreamId1); String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1); @@ -259,7 +259,7 @@ public void testTranslateComplex() { translator.translate(queryInfo.get(0), streamAppDesc, 0); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfig streamConfig = new StreamConfig(samzaConfig); + StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); @@ -296,7 +296,7 @@ public void testTranslateSubQuery() { translator.translate(queryInfo.get(0), streamAppDesc, 0); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfig streamConfig = new StreamConfig(samzaConfig); + StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); @@ -631,7 +631,7 @@ public void testTranslateStreamTableInnerJoin() { translator.translate(queryInfo.get(0), streamAppDesc, 0); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfig streamConfig = new StreamConfig(samzaConfig); + StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); String input1StreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String input1System = streamConfig.getSystem(input1StreamId); String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId); @@ -699,7 +699,7 @@ public void testTranslateStreamTableLeftJoin() { OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfig streamConfig = new StreamConfig(samzaConfig); + StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); String input1StreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String input1System = streamConfig.getSystem(input1StreamId); String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId); @@ -766,7 +766,7 @@ public void testTranslateStreamTableRightJoin() { OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfig streamConfig = new StreamConfig(samzaConfig); + StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); String input1StreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String input1System = streamConfig.getSystem(input1StreamId); String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId); diff --git a/samza-test/src/main/java/org/apache/samza/test/framework/TestRunner.java b/samza-test/src/main/java/org/apache/samza/test/framework/TestRunner.java index 082b7278c2..9121662000 100644 --- a/samza-test/src/main/java/org/apache/samza/test/framework/TestRunner.java +++ b/samza-test/src/main/java/org/apache/samza/test/framework/TestRunner.java @@ -40,7 +40,7 @@ import org.apache.samza.config.JobConfig; import org.apache.samza.config.JobCoordinatorConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.config.TaskConfig; import org.apache.samza.container.grouper.task.SingleContainerGrouperFactory; import org.apache.samza.context.ExternalContext; @@ -431,9 +431,9 @@ private void deleteDirectory(String path) { * over {@link org.apache.samza.application.descriptors.ApplicationDescriptor} generated configs */ private void addSerdeConfigs(StreamDescriptor descriptor) { - String streamIdPrefix = String.format(StreamConfig.STREAM_ID_PREFIX(), descriptor.getStreamId()); - String keySerdeConfigKey = streamIdPrefix + StreamConfig.KEY_SERDE(); - String msgSerdeConfigKey = streamIdPrefix + StreamConfig.MSG_SERDE(); + String streamIdPrefix = String.format(StreamConfig1.STREAM_ID_PREFIX(), descriptor.getStreamId()); + String keySerdeConfigKey = streamIdPrefix + StreamConfig1.KEY_SERDE(); + String msgSerdeConfigKey = streamIdPrefix + StreamConfig1.MSG_SERDE(); this.configs.put(keySerdeConfigKey, null); this.configs.put(msgSerdeConfigKey, null); } diff --git a/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java b/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java index 4f5d510511..7368ea9f59 100644 --- a/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java +++ b/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java @@ -36,7 +36,7 @@ import org.apache.samza.application.descriptors.StreamApplicationDescriptor; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.config.TaskConfig; import org.apache.samza.operators.OutputStream; import org.apache.samza.serializers.NoOpSerde; @@ -105,7 +105,7 @@ public void testDownstreamOperatorExceptionIsBubbledUp() { } private List runTest(List pageViews, Map configs) { - configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), PAGE_VIEW_STREAM), TEST_SYSTEM); + configs.put(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), PAGE_VIEW_STREAM), TEST_SYSTEM); InMemorySystemDescriptor isd = new InMemorySystemDescriptor(TEST_SYSTEM); InMemoryInputDescriptor pageViewStreamDesc = isd diff --git a/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTableWithSideInputsEndToEnd.java b/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTableWithSideInputsEndToEnd.java index 2490707742..4f9dacd624 100644 --- a/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTableWithSideInputsEndToEnd.java +++ b/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTableWithSideInputsEndToEnd.java @@ -29,7 +29,7 @@ import org.apache.samza.SamzaException; import org.apache.samza.application.StreamApplication; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; +import org.apache.samza.config.StreamConfig1; import org.apache.samza.application.descriptors.StreamApplicationDescriptor; import org.apache.samza.operators.KV; import org.apache.samza.table.descriptors.TableDescriptor; @@ -82,9 +82,9 @@ public void testJoinWithDurableSideInputTable() { private void runTest(String systemName, StreamApplication app, List pageViews, List profiles) { Map configs = new HashMap<>(); - configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), PAGEVIEW_STREAM), systemName); - configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), PROFILE_STREAM), systemName); - configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), ENRICHED_PAGEVIEW_STREAM), systemName); + configs.put(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), PAGEVIEW_STREAM), systemName); + configs.put(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), PROFILE_STREAM), systemName); + configs.put(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), ENRICHED_PAGEVIEW_STREAM), systemName); InMemorySystemDescriptor isd = new InMemorySystemDescriptor(systemName); From ad9512295cfb811749c7979d3a2575462a2c97d7 Mon Sep 17 00:00:00 2001 From: Boris S Date: Thu, 26 Sep 2019 13:32:22 -0700 Subject: [PATCH 2/9] replaced StreamConfig with StreamConfigJava --- .../system/kinesis/KinesisSystemFactory.java | 5 +- .../samza/system/eventhub/EventHubConfig.java | 6 +- .../producer/AsyncSystemProducer.java | 4 +- .../samza/config/DefaultChooserConfig.java | 4 +- .../org/apache/samza/config/SystemConfig.java | 4 +- .../samza/execution/ExecutionPlanner.java | 6 +- .../JobNodeConfigurationGenerator.java | 12 +- .../apache/samza/execution/JobPlanner.java | 4 +- .../apache/samza/execution/StreamEdge.java | 28 +- .../apache/samza/execution/StreamManager.java | 4 +- .../operators/impl/OperatorImplGraph.java | 10 +- .../samza/storage/StorageManagerUtil.java | 4 +- .../storage/TaskSideInputStorageManager.java | 4 +- .../org/apache/samza/util/StreamUtil.java | 16 +- .../samza/checkpoint/OffsetManager.scala | 12 +- .../apache/samza/config/StreamConfig1.scala | 381 ------------- .../samza/container/SamzaContainer.scala | 11 +- .../apache/samza/container/TaskInstance.scala | 5 +- .../org/apache/samza/job/JobRunner.scala | 2 +- .../MetricsSnapshotReporterFactory.scala | 9 +- .../apache/samza/config/TestStreamConfig.java | 501 +++++++++--------- .../apache/samza/config/TestSystemConfig.java | 23 +- .../samza/execution/TestStreamEdge.java | 8 +- .../operators/impl/TestOperatorImplGraph.java | 4 +- .../org/apache/samza/util/TestStreamUtil.java | 125 ++--- .../samza/system/kafka/KafkaSystemAdmin.java | 33 +- .../system/kafka/KafkaSystemFactory.scala | 2 +- .../samza/config/Log4jSystemConfig.java | 8 +- .../samza/config/Log4jSystemConfig.java | 8 +- .../samza/sql/interfaces/SqlIOConfig.java | 8 +- .../sql/translator/TestQueryTranslator.java | 20 +- .../samza/test/framework/TestRunner.java | 8 +- .../samza/test/operator/TestAsyncFlatMap.java | 4 +- .../TestLocalTableWithSideInputsEndToEnd.java | 8 +- 34 files changed, 456 insertions(+), 835 deletions(-) delete mode 100644 samza-core/src/main/scala/org/apache/samza/config/StreamConfig1.scala diff --git a/samza-aws/src/main/java/org/apache/samza/system/kinesis/KinesisSystemFactory.java b/samza-aws/src/main/java/org/apache/samza/system/kinesis/KinesisSystemFactory.java index dc7c3c71fe..09349e41f9 100644 --- a/samza-aws/src/main/java/org/apache/samza/system/kinesis/KinesisSystemFactory.java +++ b/samza-aws/src/main/java/org/apache/samza/system/kinesis/KinesisSystemFactory.java @@ -22,7 +22,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.ConfigException; import org.apache.samza.config.JobConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.config.TaskConfig; import org.apache.samza.container.grouper.stream.AllSspToSingleTaskGrouperFactory; import org.apache.samza.metrics.MetricsRegistry; @@ -31,7 +31,6 @@ import org.apache.samza.system.SystemFactory; import org.apache.samza.system.SystemProducer; import org.apache.samza.system.SystemStream; - import org.apache.samza.system.kinesis.consumer.KinesisSystemConsumer; @@ -77,7 +76,7 @@ protected void validateConfig(String system, Config config) { // Kinesis streams cannot be configured as bootstrap streams KinesisConfig kConfig = new KinesisConfig(config); kConfig.getKinesisStreams(system).forEach(stream -> { - StreamConfig1 streamConfig = new StreamConfig1(kConfig); + StreamConfigJava streamConfig = new StreamConfigJava(kConfig); SystemStream ss = new SystemStream(system, stream); if (streamConfig.getBootstrapEnabled(ss)) { throw new ConfigException("Kinesis streams cannot be configured as bootstrap streams."); diff --git a/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java b/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java index d6f67db63e..e755cb58aa 100644 --- a/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java +++ b/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java @@ -24,7 +24,7 @@ import org.apache.samza.SamzaException; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.system.eventhub.producer.EventHubSystemProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,10 +106,10 @@ public EventHubConfig(Config config) { super(config); // Build reverse index for streamName -> streamId - StreamConfig1 streamConfig = new StreamConfig1(config); + StreamConfigJava streamConfig = new StreamConfigJava(config); LOG.info("Building mappings from physicalName to streamId"); - JavaConversions.asJavaCollection(streamConfig.getStreamIds()) + streamConfig.getStreamIds() .forEach((streamId) -> { String physicalName = streamConfig.getPhysicalName(streamId); LOG.info("Obtained physicalName: {} for streamId: {} ", physicalName, streamId); diff --git a/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/AsyncSystemProducer.java b/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/AsyncSystemProducer.java index c0bd44eb08..0355227b83 100644 --- a/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/AsyncSystemProducer.java +++ b/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/AsyncSystemProducer.java @@ -34,7 +34,7 @@ import java.util.stream.Collectors; import org.apache.samza.SamzaException; import org.apache.samza.config.Config; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.metrics.Counter; import org.apache.samza.metrics.MetricsRegistry; import org.apache.samza.system.OutgoingMessageEnvelope; @@ -109,7 +109,7 @@ public abstract class AsyncSystemProducer implements SystemProducer { * @param metricsRegistry the registry */ public AsyncSystemProducer(String systemName, Config config, MetricsRegistry metricsRegistry) { - StreamConfig1 sconfig = new StreamConfig1(config); + StreamConfigJava sconfig = new StreamConfigJava(config); streamIds = config.getList(String.format(CONFIG_STREAM_LIST, systemName)); physicalToStreamIds = streamIds.stream().collect(Collectors.toMap(sconfig::getPhysicalName, Function.identity())); diff --git a/samza-core/src/main/java/org/apache/samza/config/DefaultChooserConfig.java b/samza-core/src/main/java/org/apache/samza/config/DefaultChooserConfig.java index ebf9530f62..1d6090750a 100644 --- a/samza-core/src/main/java/org/apache/samza/config/DefaultChooserConfig.java +++ b/samza-core/src/main/java/org/apache/samza/config/DefaultChooserConfig.java @@ -33,12 +33,12 @@ public class DefaultChooserConfig extends MapConfig { private static final String BATCH_SIZE = "task.consumer.batch.size"; private final TaskConfig taskConfig; - private final StreamConfig1 streamConfig; + private final StreamConfigJava streamConfig; public DefaultChooserConfig(Config config) { super(config); taskConfig = new TaskConfig(config); - streamConfig = new StreamConfig1(config); + streamConfig = new StreamConfigJava(config); } /** diff --git a/samza-core/src/main/java/org/apache/samza/config/SystemConfig.java b/samza-core/src/main/java/org/apache/samza/config/SystemConfig.java index 7e3767b810..74c4cf2396 100644 --- a/samza-core/src/main/java/org/apache/samza/config/SystemConfig.java +++ b/samza-core/src/main/java/org/apache/samza/config/SystemConfig.java @@ -160,7 +160,7 @@ public String getSystemOffsetDefault(String systemName) { * @return the key serde for the {@code systemName}, or empty if it was not found */ public Optional getSystemKeySerde(String systemName) { - return getSystemDefaultStreamProperty(systemName, StreamConfig1.KEY_SERDE()); + return getSystemDefaultStreamProperty(systemName, StreamConfigJava.KEY_SERDE); } /** @@ -168,7 +168,7 @@ public Optional getSystemKeySerde(String systemName) { * @return the message serde for the {@code systemName}, or empty if it was not found */ public Optional getSystemMsgSerde(String systemName) { - return getSystemDefaultStreamProperty(systemName, StreamConfig1.MSG_SERDE()); + return getSystemDefaultStreamProperty(systemName, StreamConfigJava.MSG_SERDE); } /** diff --git a/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java b/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java index 58ae1aa150..3ea2b36528 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java +++ b/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java @@ -44,7 +44,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.operators.spec.InputOperatorSpec; import org.apache.samza.operators.spec.OperatorSpec; import org.apache.samza.operators.spec.StreamTableJoinOperatorSpec; @@ -68,12 +68,12 @@ public class ExecutionPlanner { private final Config config; private final StreamManager streamManager; - private final StreamConfig1 streamConfig; + private final StreamConfigJava streamConfig; public ExecutionPlanner(Config config, StreamManager streamManager) { this.config = config; this.streamManager = streamManager; - this.streamConfig = new StreamConfig1(config); + this.streamConfig = new StreamConfigJava(config); } public ExecutionPlan plan(ApplicationDescriptorImpl appDesc) { diff --git a/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java b/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java index ce8feb8fe9..4d78ea7d28 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java +++ b/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java @@ -39,7 +39,7 @@ import org.apache.samza.config.MapConfig; import org.apache.samza.config.SerializerConfig; import org.apache.samza.config.StorageConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.config.TaskConfig; import org.apache.samza.operators.KV; import org.apache.samza.operators.spec.JoinOperatorSpec; @@ -251,7 +251,7 @@ private void configureTables(Map generatedConfig, Config origina .map(sideInput -> StreamUtil.getSystemStreamFromNameOrId(originalConfig, sideInput)) .forEach(systemStream -> { inputs.add(StreamUtil.getNameFromSystemStream(systemStream)); - generatedConfig.put(String.format(StreamConfig1.STREAM_PREFIX() + StreamConfig1.BOOTSTRAP(), + generatedConfig.put(String.format(StreamConfigJava.STREAM_PREFIX() + StreamConfigJava.BOOTSTRAP(), systemStream.getSystem(), systemStream.getStream()), "true"); }); } @@ -314,14 +314,14 @@ private void configureSerdes(Map configs, Map { - String streamIdPrefix = String.format(StreamConfig1.STREAM_ID_PREFIX(), streamId); - String keySerdeConfigKey = streamIdPrefix + StreamConfig1.KEY_SERDE(); + String streamIdPrefix = String.format(StreamConfigJava.STREAM_ID_PREFIX(), streamId); + String keySerdeConfigKey = streamIdPrefix + StreamConfigJava.KEY_SERDE(); configs.put(keySerdeConfigKey, serdeUUIDs.get(serde)); }); streamMsgSerdes.forEach((streamId, serde) -> { - String streamIdPrefix = String.format(StreamConfig1.STREAM_ID_PREFIX(), streamId); - String valueSerdeConfigKey = streamIdPrefix + StreamConfig1.MSG_SERDE(); + String streamIdPrefix = String.format(StreamConfigJava.STREAM_ID_PREFIX(), streamId); + String valueSerdeConfigKey = streamIdPrefix + StreamConfigJava.MSG_SERDE(); configs.put(valueSerdeConfigKey, serdeUUIDs.get(serde)); }); diff --git a/samza-core/src/main/java/org/apache/samza/execution/JobPlanner.java b/samza-core/src/main/java/org/apache/samza/execution/JobPlanner.java index 8c00040ab1..3f4c79e78b 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/JobPlanner.java +++ b/samza-core/src/main/java/org/apache/samza/execution/JobPlanner.java @@ -35,7 +35,7 @@ import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; import org.apache.samza.config.ShellCommandConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.config.TaskConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,7 +126,7 @@ private Map getGeneratedConfig(String runId) { generatedConfig.put(ApplicationConfig.APP_RUN_ID, runId); } - StreamConfig1 streamConfig = new StreamConfig1(userConfig); + StreamConfigJava streamConfig = new StreamConfigJava(userConfig); Set inputStreamIds = new HashSet<>(appDesc.getInputStreamIds()); inputStreamIds.removeAll(appDesc.getOutputStreamIds()); // exclude intermediate streams ApplicationConfig.ApplicationMode mode = diff --git a/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java b/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java index 50d6004ab3..c3d87bd769 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java +++ b/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java @@ -19,19 +19,19 @@ package org.apache.samza.execution; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.apache.samza.config.ApplicationConfig; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.system.StreamSpec; import org.apache.samza.system.SystemStream; import org.apache.samza.util.StreamUtil; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + /** * A StreamEdge connects the source {@link JobNode}s to the target {@link JobNode}s with a stream. @@ -56,7 +56,7 @@ public class StreamEdge { this.isIntermediate = isIntermediate; // broadcast can be configured either by an operator or via the configs this.isBroadcast = - isBroadcast || new StreamConfig1(config).getBroadcastEnabled(streamSpec.toSystemStream()); + isBroadcast || new StreamConfigJava(config).getBroadcastEnabled(streamSpec.toSystemStream()); this.config = config; if (isBroadcast && isIntermediate) { partitions = 1; @@ -122,21 +122,21 @@ Config generateConfig() { Map streamConfig = new HashMap<>(); StreamSpec spec = getStreamSpec(); String streamId = spec.getId(); - streamConfig.put(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), streamId), spec.getSystemName()); - streamConfig.put(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), spec.getPhysicalName()); + streamConfig.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID(), streamId), spec.getSystemName()); + streamConfig.put(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), spec.getPhysicalName()); if (isIntermediate()) { - streamConfig.put(String.format(StreamConfig1.IS_INTERMEDIATE_FOR_STREAM_ID(), streamId), "true"); - streamConfig.put(String.format(StreamConfig1.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), streamId), "true"); + streamConfig.put(String.format(StreamConfigJava.IS_INTERMEDIATE_FOR_STREAM_ID(), streamId), "true"); + streamConfig.put(String.format(StreamConfigJava.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), streamId), "true"); // Setting offset.default to oldest only if the job is running in batch mode if (ApplicationConfig.ApplicationMode.BATCH.equals(new ApplicationConfig(config).getAppMode())) { - streamConfig.put(String.format(StreamConfig1.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), streamId), "oldest"); + streamConfig.put(String.format(StreamConfigJava.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), streamId), "oldest"); } - streamConfig.put(String.format(StreamConfig1.PRIORITY_FOR_STREAM_ID(), streamId), String.valueOf(Integer.MAX_VALUE)); + streamConfig.put(String.format(StreamConfigJava.PRIORITY_FOR_STREAM_ID(), streamId), String.valueOf(Integer.MAX_VALUE)); } spec.getConfig().forEach((property, value) -> { - streamConfig.put(String.format(StreamConfig1.STREAM_ID_PREFIX(), streamId) + property, value); + streamConfig.put(String.format(StreamConfigJava.STREAM_ID_PREFIX(), streamId) + property, value); }); return new MapConfig(streamConfig); diff --git a/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java b/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java index aec1a467eb..5429847cfa 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java +++ b/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java @@ -110,10 +110,10 @@ public void clearStreamsFromPreviousRun(Config prevConfig, ClassLoader classLoad ApplicationConfig appConfig = new ApplicationConfig(prevConfig); LOGGER.info("run.id from previous run is {}", appConfig.getRunId()); - StreamConfig1 streamConfig = new StreamConfig1(prevConfig); + StreamConfigJava streamConfig = new StreamConfigJava(prevConfig); //Find all intermediate streams and clean up - Set intStreams = JavaConversions.asJavaCollection(streamConfig.getStreamIds()).stream() + Set intStreams = streamConfig.getStreamIds().stream() .filter(streamConfig::getIsIntermediateStream) .map(id -> new StreamSpec(id, streamConfig.getPhysicalName(id), streamConfig.getSystem(id))) .collect(Collectors.toSet()); diff --git a/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java b/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java index f89b67f7bd..86cd375083 100644 --- a/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java +++ b/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java @@ -22,7 +22,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import org.apache.samza.config.Config; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.context.Context; import org.apache.samza.context.InternalTaskContext; import org.apache.samza.job.model.JobModel; @@ -99,7 +99,7 @@ public class OperatorImplGraph { */ public OperatorImplGraph(OperatorSpecGraph specGraph, Context context, Clock clock) { this.clock = clock; - StreamConfig1 streamConfig = new StreamConfig1(context.getJobContext().getConfig()); + StreamConfigJava streamConfig = new StreamConfigJava(context.getJobContext().getConfig()); this.internalTaskContext = new InternalTaskContext(context); Map producerTaskCounts = hasIntermediateStreams(specGraph) @@ -215,7 +215,7 @@ private OperatorImpl createAndRegisterOperatorImpl(OperatorSpec prevOperatorSpec */ OperatorImpl createOperatorImpl(OperatorSpec prevOperatorSpec, OperatorSpec operatorSpec, Context context) { Config config = context.getJobContext().getConfig(); - StreamConfig1 streamConfig = new StreamConfig1(config); + StreamConfigJava streamConfig = new StreamConfigJava(config); if (operatorSpec instanceof InputOperatorSpec) { return new InputOperatorImpl((InputOperatorSpec) operatorSpec); } else if (operatorSpec instanceof StreamOperatorSpec) { @@ -380,7 +380,7 @@ static Multimap getStreamToConsumerTasks(JobModel jobModel * @return mapping from output streams to input streams */ static Multimap getIntermediateToInputStreamsMap( - OperatorSpecGraph specGraph, StreamConfig1 streamConfig) { + OperatorSpecGraph specGraph, StreamConfigJava streamConfig) { Multimap outputToInputStreams = HashMultimap.create(); specGraph.getInputOperators().entrySet().stream() .forEach(entry -> { @@ -391,7 +391,7 @@ static Multimap getIntermediateToInputStreamsMap( } private static void computeOutputToInput(SystemStream input, OperatorSpec opSpec, - Multimap outputToInputStreams, StreamConfig1 streamConfig) { + Multimap outputToInputStreams, StreamConfigJava streamConfig) { if (opSpec instanceof PartitionByOperatorSpec) { PartitionByOperatorSpec spec = (PartitionByOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); diff --git a/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java b/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java index e60304d2a8..09a3ff230c 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java +++ b/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java @@ -57,8 +57,8 @@ public class StorageManagerUtil { /** * Fetch the starting offset for the input {@link SystemStreamPartition} * - * Note: The method doesn't respect {@link org.apache.samza.config.StreamConfig1#CONSUMER_OFFSET_DEFAULT()} and - * {@link org.apache.samza.config.StreamConfig1#CONSUMER_RESET_OFFSET()} configurations. It will use the locally + * Note: The method doesn't respect {@link org.apache.samza.config.StreamConfigJava#CONSUMER_OFFSET_DEFAULT} and + * {@link org.apache.samza.config.StreamConfigJava#CONSUMER_RESET_OFFSET} configurations. It will use the locally * checkpointed offset if it is valid, or fall back to oldest offset of the stream. * * @param ssp system stream partition for which starting offset is requested diff --git a/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java b/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java index 029d504c2d..41142c5495 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java +++ b/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java @@ -162,8 +162,8 @@ public StorageEngine getStore(String storeName) { /** * Gets the starting offset for the given side input {@link SystemStreamPartition}. * - * Note: The method doesn't respect {@link org.apache.samza.config.StreamConfig1#CONSUMER_OFFSET_DEFAULT()} and - * {@link org.apache.samza.config.StreamConfig1#CONSUMER_RESET_OFFSET()} configurations. It will use the local offset + * Note: The method doesn't respect {@link org.apache.samza.config.StreamConfigJava#CONSUMER_OFFSET_DEFAULT} and + * {@link org.apache.samza.config.StreamConfigJava#CONSUMER_RESET_OFFSET} configurations. It will use the local offset * file if it is valid, else it will fall back to oldest offset in the stream. * * @param ssp side input system stream partition to get the starting offset for diff --git a/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java b/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java index 3f41406085..14b57bfb1a 100644 --- a/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java +++ b/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java @@ -18,15 +18,17 @@ */ package org.apache.samza.util; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; import org.apache.samza.SamzaException; import org.apache.samza.config.Config; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.system.StreamSpec; import org.apache.samza.system.SystemStream; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + public class StreamUtil { /** * Gets the {@link SystemStream} corresponding to the provided stream, which may be @@ -43,7 +45,7 @@ public static SystemStream getSystemStreamFromNameOrId(Config config, String str String.format("Invalid stream %s. Expected to be of the format streamId or systemName.streamName", stream)); } if (parts.length == 1) { - return new StreamConfig1(config).streamIdToSystemStream(stream); + return new StreamConfigJava(config).streamIdToSystemStream(stream); } else { return new SystemStream(parts[0], parts[1]); } @@ -78,11 +80,11 @@ public static String getNameFromSystemStream(SystemStream systemStream) { return systemStream.getSystem() + "." + systemStream.getStream(); } - public static Set getStreamSpecs(Set streamIds, StreamConfig1 streamConfig) { + public static Set getStreamSpecs(Set streamIds, StreamConfigJava streamConfig) { return streamIds.stream().map(streamId -> getStreamSpec(streamId, streamConfig)).collect(Collectors.toSet()); } - public static StreamSpec getStreamSpec(String streamId, StreamConfig1 streamConfig) { + public static StreamSpec getStreamSpec(String streamId, StreamConfigJava streamConfig) { String physicalName = streamConfig.getPhysicalName(streamId); String system = streamConfig.getSystem(streamId); Map streamProperties = streamConfig.getStreamProperties(streamId); diff --git a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala index 3f014d8f04..4cb6ce7bce 100644 --- a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala +++ b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala @@ -80,16 +80,16 @@ object OffsetManager extends Logging { offsetManagerMetrics: OffsetManagerMetrics = new OffsetManagerMetrics) = { debug("Building offset manager for %s." format systemStreamMetadata) + val streamConfig = new StreamConfigJava(config) + val offsetSettings = systemStreamMetadata .map { case (systemStream, systemStreamMetadata) => // Get default offset. - val streamDefaultOffset = config.getDefaultStreamOffset(systemStream) - val systemDefaultOffset = new SystemConfig(config).getSystemOffsetDefault(systemStream.getSystem) - val defaultOffsetType = if (streamDefaultOffset.isDefined) { - OffsetType.valueOf(streamDefaultOffset.get.toUpperCase) - } else if (systemDefaultOffset != null) { - OffsetType.valueOf(systemDefaultOffset.toUpperCase) + val streamDefaultOffset = streamConfig.getDefaultStreamOffset(systemStream) + val systemDefaultOffset = new SystemConfig(streamConfig).getSystemOffsetDefault(systemStream.getSystem) + val defaultOffsetType = if (streamDefaultOffset != null) { + OffsetType.valueOf(streamDefaultOffset.toUpperCase) } else { info("No default offset for %s defined. Using upcoming." format systemStream) OffsetType.UPCOMING diff --git a/samza-core/src/main/scala/org/apache/samza/config/StreamConfig1.scala b/samza-core/src/main/scala/org/apache/samza/config/StreamConfig1.scala deleted file mode 100644 index 6376c51db5..0000000000 --- a/samza-core/src/main/scala/org/apache/samza/config/StreamConfig1.scala +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.samza.config - -import org.apache.samza.system.SystemStream -import org.apache.samza.util.Logging - -import scala.collection.JavaConverters._ - -object StreamConfig1 { - // Samza configs for streams - val SAMZA_PROPERTY = "samza." - val SYSTEM = SAMZA_PROPERTY + "system" - val PHYSICAL_NAME = SAMZA_PROPERTY + "physical.name" - val MSG_SERDE = SAMZA_PROPERTY + "msg.serde" - val KEY_SERDE = SAMZA_PROPERTY + "key.serde" - val CONSUMER_RESET_OFFSET = SAMZA_PROPERTY + "reset.offset" - val CONSUMER_OFFSET_DEFAULT = SAMZA_PROPERTY + "offset.default" - val BOOTSTRAP = SAMZA_PROPERTY + "bootstrap" - val PRIORITY = SAMZA_PROPERTY + "priority" - val IS_INTERMEDIATE = SAMZA_PROPERTY + "intermediate" - val DELETE_COMMITTED_MESSAGES = SAMZA_PROPERTY + "delete.committed.messages" - val IS_BOUNDED = SAMZA_PROPERTY + "bounded" - val BROADCAST = SAMZA_PROPERTY + "broadcast" - - // We don't want any external dependencies on these patterns while both exist. Use getProperty to ensure proper values. - private val STREAMS_PREFIX = "streams." - - val STREAM_PREFIX = "systems.%s.streams.%s." - val STREAM_ID_PREFIX = STREAMS_PREFIX + "%s." - val SYSTEM_FOR_STREAM_ID = STREAM_ID_PREFIX + SYSTEM - val PHYSICAL_NAME_FOR_STREAM_ID = STREAM_ID_PREFIX + PHYSICAL_NAME - val IS_INTERMEDIATE_FOR_STREAM_ID = STREAM_ID_PREFIX + IS_INTERMEDIATE - val DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID = STREAM_ID_PREFIX + DELETE_COMMITTED_MESSAGES - val IS_BOUNDED_FOR_STREAM_ID = STREAM_ID_PREFIX + IS_BOUNDED - val PRIORITY_FOR_STREAM_ID = STREAM_ID_PREFIX + PRIORITY - val CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID = STREAM_ID_PREFIX + CONSUMER_OFFSET_DEFAULT - val BOOTSTRAP_FOR_STREAM_ID = STREAM_ID_PREFIX + BOOTSTRAP - val BROADCAST_FOR_STREAM_ID = STREAM_ID_PREFIX + BROADCAST - - implicit def Config2Stream(config: Config) = new StreamConfig1(config) -} - -/** - * Helper for accessing configs related to stream properties. - * - * For most configs, this currently supports two different formats for specifying stream properties: - * 1) "streams.{streamId}.{property}" (recommended to use this format) - * 2) "systems.{systemName}.streams.{streamName}.{property}" (legacy) - * Note that some config lookups are only supported through the "streams.{streamId}.{property}". See the specific - * accessor method to determine which formats are supported. - * - * Summary of terms: - * - streamId: logical identifier used for a stream; configs are specified using this streamId - * - physical stream: concrete name for a stream (if the physical stream is not explicitly configured, then the streamId - * is used as the physical stream - * - streamName: within the javadoc for this class, streamName is the same as physical stream - * - samza property: property which is Samza-specific, which will have "samza." as a prefix (e.g. "samza.key.serde"); - * this is in contrast to stream-specific properties which are related to specific stream technologies - */ -class StreamConfig1(config: Config) extends ScalaMapConfig(config) with Logging { - def getStreamMsgSerde(systemStream: SystemStream) = nonEmptyOption(getSamzaProperty(systemStream, StreamConfig1.MSG_SERDE)) - - def getStreamKeySerde(systemStream: SystemStream) = nonEmptyOption(getSamzaProperty(systemStream, StreamConfig1.KEY_SERDE)) - - def getResetOffset(systemStream: SystemStream) = - Option(getSamzaProperty(systemStream, StreamConfig1.CONSUMER_RESET_OFFSET)) match { - case Some("true") => true - case Some("false") => false - case Some(resetOffset) => - warn("Got a .samza.reset.offset configuration for SystemStream %s that is not true or false (was %s). Defaulting to false." - format (systemStream.toString format (systemStream.getSystem, systemStream.getStream), resetOffset)) - false - case _ => false - } - - def isResetOffsetConfigured(systemStream: SystemStream) = - containsSamzaProperty(systemStream, StreamConfig1.CONSUMER_RESET_OFFSET) - - def getDefaultStreamOffset(systemStream: SystemStream) = - Option(getSamzaProperty(systemStream, StreamConfig1.CONSUMER_OFFSET_DEFAULT)) - - def isDefaultStreamOffsetConfigured(systemStream: SystemStream) = - containsSamzaProperty(systemStream, StreamConfig1.CONSUMER_OFFSET_DEFAULT) - - def getBootstrapEnabled(systemStream: SystemStream) = - java.lang.Boolean.parseBoolean(getSamzaProperty(systemStream, StreamConfig1.BOOTSTRAP)) - - def getBroadcastEnabled(systemStream: SystemStream) = - java.lang.Boolean.parseBoolean(getSamzaProperty(systemStream, StreamConfig1.BROADCAST)) - - def getPriority(systemStream: SystemStream) = - java.lang.Integer.parseInt(getSamzaProperty(systemStream, StreamConfig1.PRIORITY, "-1")) - - /** - * Returns a list of all SystemStreams that have a serde defined from the config file. - */ - def getSerdeStreams(systemName: String) = { - val subConf = config.subset("systems.%s.streams." format systemName, true) - val legacySystemStreams = subConf - .asScala - .keys - .filter(k => k.endsWith(StreamConfig1.MSG_SERDE) || k.endsWith(StreamConfig1.KEY_SERDE)) - .map(k => { - val streamName = k.substring(0, k.length - 16 /* .samza.XXX.serde length */ ) - new SystemStream(systemName, streamName) - }).toSet - - val systemStreams = subset(StreamConfig1.STREAMS_PREFIX) - .asScala - .keys - .filter(k => k.endsWith(StreamConfig1.MSG_SERDE) || k.endsWith(StreamConfig1.KEY_SERDE)) - .map(k => k.substring(0, k.length - 16 /* .samza.XXX.serde length */ )) - .filter(streamId => systemName.equals(getSystem(streamId))) - .map(streamId => streamIdToSystemStream(streamId)).toSet - - legacySystemStreams.union(systemStreams) - } - - /** - * Gets the properties for the streamId which are not Samza properties (i.e. do not have a "samza." prefix). This - * includes current and legacy config styles. - * This will return a Config with the properties that match the following formats (if a property is specified through - * multiple formats, priority is top to bottom): - * 1) "streams.{streamId}.{property}" - * 2) "systems.{systemName}.streams.{streamName}.{property}" where systemName is the system mapped to the streamId in - * the config and streamName is the physical stream name mapped to the stream id - * 3) "systems.{systemName}.default.stream.{property}" where systemName is the system mapped to the streamId in the - * config - * - * @param streamId the identifier for the stream in the config. - * @return the merged map of config properties from both the legacy and new config styles - */ - def getStreamProperties(streamId: String) = { - val allProperties = getAllStreamProperties(streamId) - val samzaProperties = allProperties.subset(StreamConfig1.SAMZA_PROPERTY, false) - val filteredStreamProperties: java.util.Map[String, String] = - allProperties.asScala.filterKeys(k => !samzaProperties.containsKey(k)).asJava - new MapConfig(filteredStreamProperties) - } - - /** - * Gets the System associated with the specified streamId. - * It first looks for the property - * streams.{streamId}.system - * - * If no value was provided, it uses - * job.default.system - * - * @param streamId the identifier for the stream in the config. - * @return the system name associated with the stream. - */ - def getSystem(streamId: String) = { - getOption(StreamConfig1.SYSTEM_FOR_STREAM_ID format streamId) match { - case Some(system) => system - case _ => new JobConfig(config).getDefaultSystem().orElse(null) - } - } - - /** - * Gets the physical name for the specified streamId. - * - * @param streamId the identifier for the stream in the config. - * @return the physical identifier for the stream or the default if it is undefined. - */ - def getPhysicalName(streamId: String) = { - // use streamId as the default physical name - get(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID format streamId, streamId) - } - - /** - * Gets the boolean flag of whether the specified streamId is an intermediate stream - * @param streamId the identifier for the stream in the config. - * @return true if the stream is intermediate - */ - def getIsIntermediateStream(streamId: String) = { - getBoolean(StreamConfig1.IS_INTERMEDIATE_FOR_STREAM_ID format streamId, false) - } - - /** - * Gets the boolean flag of whether the committed messages specified streamId can be deleted - * @param streamId the identifier for the stream in the config. - * @return true if the committed messages of the stream can be deleted - */ - def getDeleteCommittedMessages(streamId: String) = { - getBoolean(StreamConfig1.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID format streamId, false) - } - - def getIsBounded(streamId: String) = { - getBoolean(StreamConfig1.IS_BOUNDED_FOR_STREAM_ID format streamId, false) - } - - /** - * Gets the stream IDs of all the streams defined in the config - * @return collection of stream IDs - */ - def getStreamIds(): Iterable[String] = { - // StreamIds are not allowed to have '.' so the first index of '.' marks the end of the streamId. - subset(StreamConfig1.STREAMS_PREFIX).asScala.keys.map(key => key.substring(0, key.indexOf("."))) - } - - /** - * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza - * interacts with the stream, as opposed to a property of the stream itself. - * - * First, tries to map the systemStream to a streamId. This will only find a streamId if the stream is a physical name - * (explicitly mapped physical name or a stream id without a physical name mapping). That means this will not map a - * stream id to itself if there is a mapping from the stream id to a physical stream name. This also requires that the - * stream id is mapped to a system in the config. - * If a stream id is found: - * 1) Look for "streams.{streamId}.{property}" for the stream id. - * 2) Otherwise, look for "systems.{systemName}.streams.{streamName}.{property}" in which the systemName is the system - * mapped to the stream id and the streamName is the physical stream name for the stream id. - * 3) Otherwise, look for "systems.{systemName}.default.stream.{property}" in which the systemName is the system - * mapped to the stream id. - * If a stream id was not found or no property could be found using the above keys: - * 1) Look for "systems.{systemName}.streams.{streamName}.{property}" in which the systemName is the system in the - * input systemStream and the streamName is the stream from the input systemStream. - * 2) Otherwise, look for "systems.{systemName}.default.stream.{property}" in which the systemName is the system - * in the input systemStream. - * - * @param systemStream the SystemStream for which the property value will be retrieved. - * @param property the samza property key (including the "samza." prefix); for example, for both - * "streams.streamId.samza.prop.key" and "systems.system.streams.streamName.samza.prop.key", this - * argument should have the value "samza.prop.key" - */ - private def getSamzaProperty(systemStream: SystemStream, property: String): String = { - if (!property.startsWith(StreamConfig1.SAMZA_PROPERTY)) { - throw new IllegalArgumentException( - "Attempt to fetch a non samza property for SystemStream %s named %s" format(systemStream, property)) - } - - val streamVal = getAllStreamProperties(systemStreamToStreamId(systemStream)).get(property) - if (streamVal != null) { - streamVal - } else { - getSystemStreamProperties(systemStream.getSystem(), systemStream.getStream).get(property) - } - } - - /** - * Gets a Samza property, with a default value used if no property value is found. - * See getSamzaProperty(SystemStream, String). - * - * @param systemStream the SystemStream for which the property value will be retrieved. - * @param property the samza property key (including the "samza." prefix); for example, for both - * "streams.streamId.samza.prop.key" and "systems.system.streams.streamName.samza.prop.key", this - * argument should have the value "samza.prop.key" - * @param defaultValue the default value to use if the property value is not found - */ - private def getSamzaProperty(systemStream: SystemStream, property: String, defaultValue: String): String = { - val streamVal = getSamzaProperty(systemStream, property) - - if (streamVal != null) { - streamVal - } else { - defaultValue - } - } - - /** - * Determines if a Samza property is specified. - * See getSamzaProperty(SystemStream, String). - * - * @param systemStream the SystemStream for the property value to check - * @param property the samza property key (including the "samza." prefix); for example, for both - * "streams.streamId.samza.prop.key" and "systems.system.streams.streamName.samza.prop.key", this - * argument should have the value "samza.prop.key" - */ - private def containsSamzaProperty(systemStream: SystemStream, property: String): Boolean = { - if (!property.startsWith(StreamConfig1.SAMZA_PROPERTY)) { - throw new IllegalArgumentException( - "Attempt to fetch a non samza property for SystemStream %s named %s" format(systemStream, property)) - } - getSamzaProperty(systemStream, property) != null - } - - - /** - * Finds the properties from the legacy config style (config key includes system). - * This will return a Config with the properties that match the following formats (if a property is specified through - * multiple formats, priority is top to bottom): - * 1) "systems.{systemName}.streams.{streamName}.{property}" - * 2) "systems.{systemName}.default.stream.{property}" - * - * @param systemName the system name under which the properties are configured - * @param streamName the stream name - * @return the map of properties for the stream - */ - private def getSystemStreamProperties(systemName: String, streamName: String) = { - if (systemName == null) { - Map() - } - val systemConfig = new SystemConfig(config) - val defaults = systemConfig.getDefaultStreamProperties(systemName) - val explicitConfigs = config.subset(StreamConfig1.STREAM_PREFIX format(systemName, streamName), true) - new MapConfig(defaults, explicitConfigs) - } - - /** - * Gets all of the properties for the specified streamId (includes current and legacy config styles). - * This will return a Config with the properties that match the following formats (if a property is specified through - * multiple formats, priority is top to bottom): - * 1) "streams.{streamId}.{property}" - * 2) "systems.{systemName}.streams.{streamName}.{property}" where systemName is the system mapped to the streamId in - * the config and streamName is the physical stream name mapped to the stream id - * 3) "systems.{systemName}.default.stream.{property}" where systemName is the system mapped to the streamId in the - * config - * - * @param streamId the identifier for the stream in the config. - * @return the merged map of config properties from both the legacy and new config styles - */ - private def getAllStreamProperties(streamId: String) = { - val allProperties = subset(StreamConfig1.STREAM_ID_PREFIX format streamId) - val inheritedLegacyProperties: java.util.Map[String, String] = - getSystemStreamProperties(getSystem(streamId), getPhysicalName(streamId)) - new MapConfig(java.util.Arrays.asList(inheritedLegacyProperties, allProperties)) - } - - private def getStreamIdsForSystem(system: String): Iterable[String] = { - getStreamIds().filter(streamId => system.equals(getSystem(streamId))) - } - - /** - * Finds the stream id which corresponds to the systemStream. - * This finds the stream id that is mapped to the system in systemStream through the config and that has a physical - * name (the physical name might be the stream id itself if there is no explicit mapping) that matches the stream in - * systemStream. - * Note: If the stream in the systemStream is a stream id which is mapped to a physical stream, then that stream won't - * be returned as a stream id here, since the stream in systemStream doesn't match the physical stream name. - * - * @param systemStream system stream to map to stream id - * @return stream id corresponding to the system stream - */ - private def systemStreamToStreamId(systemStream: SystemStream): String = { - val streamIds = getStreamIdsForSystem(systemStream.getSystem) - .filter(streamId => systemStream.getStream().equals(getPhysicalName(streamId))) - if (streamIds.size > 1) { - throw new IllegalStateException("There was more than one stream found for system stream %s" format(systemStream)) - } - - if (streamIds.isEmpty) { - null - } else { - streamIds.head - } - } - - /** - * A streamId is translated to a SystemStream by looking up its System and physicalName. It - * will use the streamId as the stream name if the physicalName doesn't exist. - */ - def streamIdToSystemStream(streamId: String): SystemStream = { - new SystemStream(getSystem(streamId), getPhysicalName(streamId)) - } - - private def nonEmptyOption(value: String): Option[String] = { - if (value == null || value.isEmpty) { - None - } else { - Some(value) - } - } -} diff --git a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala index 91498bcb53..1ffd111796 100644 --- a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala +++ b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala @@ -196,7 +196,8 @@ object SamzaContainer extends Logging { info("Got system names: %s" format systemNames) - val serdeStreams = systemNames.foldLeft(Set[SystemStream]())(_ ++ config.getSerdeStreams(_)) + val streamConfig = new StreamConfigJava(config) + val serdeStreams = systemNames.foldLeft(Set[SystemStream]())(_ ++ streamConfig.getSerdeStreams(_).asScala) info("Got serde streams: %s" format serdeStreams) @@ -327,11 +328,11 @@ object SamzaContainer extends Logging { debug("Got system message serdes: %s" format systemMessageSerdes) - val systemStreamKeySerdes = buildSystemStreamSerdeMap(systemStream => config.getStreamKeySerde(systemStream)) + val systemStreamKeySerdes = buildSystemStreamSerdeMap(systemStream => streamConfig.getStreamKeySerde(systemStream)) debug("Got system stream key serdes: %s" format systemStreamKeySerdes) - val systemStreamMessageSerdes = buildSystemStreamSerdeMap(systemStream => config.getStreamMsgSerde(systemStream)) + val systemStreamMessageSerdes = buildSystemStreamSerdeMap(systemStream => streamConfig.getStreamMsgSerde(systemStream)) debug("Got system stream message serdes: %s" format systemStreamMessageSerdes) @@ -360,9 +361,9 @@ object SamzaContainer extends Logging { SystemClock.instance, getChangelogSSPsForContainer(containerModel, changeLogSystemStreams).asJava) - val intermediateStreams = config + val intermediateStreams = streamConfig .getStreamIds - .filter(config.getIsIntermediateStream(_)) + .filter(streamConfig.getIsIntermediateStream(_)) .toList info("Got intermediate streams: %s" format intermediateStreams) diff --git a/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala b/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala index 89b27f4520..580475ca28 100644 --- a/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala +++ b/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala @@ -99,9 +99,10 @@ class TaskInstance( private val config: Config = jobContext.getConfig - val intermediateStreams: Set[String] = config.getStreamIds.filter(config.getIsIntermediateStream).toSet + val streamConfig: StreamConfigJava = new StreamConfigJava(config) + val intermediateStreams: Set[String] = streamConfig.getStreamIds.filter(streamConfig.getIsIntermediateStream).toSet - val streamsToDeleteCommittedMessages: Set[String] = config.getStreamIds.filter(config.getDeleteCommittedMessages).map(config.getPhysicalName).toSet + val streamsToDeleteCommittedMessages: Set[String] = streamConfig.getStreamIds.filter(streamConfig.getDeleteCommittedMessages).map(streamConfig.getPhysicalName).toSet def registerOffsets { debug("Registering offsets for taskName: %s" format taskName) diff --git a/samza-core/src/main/scala/org/apache/samza/job/JobRunner.scala b/samza-core/src/main/scala/org/apache/samza/job/JobRunner.scala index 3c902694b6..74fb7fbbdd 100644 --- a/samza-core/src/main/scala/org/apache/samza/job/JobRunner.scala +++ b/samza-core/src/main/scala/org/apache/samza/job/JobRunner.scala @@ -124,7 +124,7 @@ class JobRunner(config: Config) extends Logging { val diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsSystemStreamName) val diagnosticsSysAdmin = systemAdmins.getSystemAdmin(diagnosticsSystemStream.getSystem) val diagnosticsStreamSpec = new StreamSpec(DIAGNOSTICS_STREAM_ID, diagnosticsSystemStream.getStream, - diagnosticsSystemStream.getSystem, new StreamConfig1(config).getStreamProperties(DIAGNOSTICS_STREAM_ID)) + diagnosticsSystemStream.getSystem, new StreamConfigJava(config).getStreamProperties(DIAGNOSTICS_STREAM_ID)) info("Creating diagnostics stream %s" format diagnosticsSystemStream.getStream) diagnosticsSysAdmin.start() diff --git a/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala b/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala index deafe542b1..e8bdccfac4 100644 --- a/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala +++ b/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala @@ -21,7 +21,7 @@ package org.apache.samza.metrics.reporter import org.apache.samza.util.{Logging, StreamUtil, Util} import org.apache.samza.SamzaException -import org.apache.samza.config.{Config, JobConfig, MetricsConfig, SerializerConfig, SystemConfig} +import org.apache.samza.config.{Config, JobConfig, MetricsConfig, SerializerConfig, StreamConfigJava, SystemConfig} import org.apache.samza.config.StreamConfig1.Config2Stream import org.apache.samza.metrics.MetricsReporter import org.apache.samza.metrics.MetricsReporterFactory @@ -63,10 +63,11 @@ class MetricsSnapshotReporterFactory extends MetricsReporterFactory with Logging val producer = systemFactory.getProducer(systemName, config, registry) info("Got producer %s." format producer) + val streamConfig = new StreamConfigJava(config) - val streamSerdeName = config.getStreamMsgSerde(systemStream) - val systemSerdeName = JavaOptionals.toRichOptional(systemConfig.getSystemMsgSerde(systemName)).toOption - val serdeName = streamSerdeName.getOrElse(systemSerdeName.getOrElse(null)) + val streamSerdeName = streamConfig.getStreamMsgSerde(systemStream) + val systemSerdeName = systemConfig.getSystemMsgSerde(systemName) + val serdeName = streamSerdeName.orElse(systemSerdeName.orElse(null)) val serializerConfig = new SerializerConfig(config) val serde = if (serdeName != null) { JavaOptionals.toRichOptional(serializerConfig.getSerdeFactoryClass(serdeName)).toOption match { diff --git a/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java b/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java index 23dc142910..653d5b312b 100644 --- a/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java +++ b/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java @@ -47,258 +47,247 @@ public class TestStreamConfig { @Test public void testGetStreamMsgSerde() { String value = "my.msg.serde"; - doTestSamzaProperty(StreamConfig1.MSG_SERDE(), value, + doTestSamzaProperty(StreamConfigJava.MSG_SERDE, value, (config, systemStream) -> assertEquals(Option.apply(value), config.getStreamMsgSerde(systemStream))); - doTestSamzaProperty(StreamConfig1.MSG_SERDE(), "", + doTestSamzaProperty(StreamConfigJava.MSG_SERDE, "", (config, systemStream) -> assertEquals(Option.empty(), config.getStreamMsgSerde(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig1.MSG_SERDE(), + doTestSamzaPropertyDoesNotExist(StreamConfigJava.MSG_SERDE, (config, systemStream) -> assertEquals(Option.empty(), config.getStreamMsgSerde(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig1::getStreamMsgSerde); + doTestSamzaPropertyInvalidConfig(StreamConfigJava::getStreamMsgSerde); } @Test public void testGetStreamKeySerde() { String value = "my.key.serde"; - doTestSamzaProperty(StreamConfig1.KEY_SERDE(), value, + doTestSamzaProperty(StreamConfigJava.KEY_SERDE, value, (config, systemStream) -> assertEquals(Option.apply(value), config.getStreamKeySerde(systemStream))); - doTestSamzaProperty(StreamConfig1.KEY_SERDE(), "", + doTestSamzaProperty(StreamConfigJava.KEY_SERDE, "", (config, systemStream) -> assertEquals(Option.empty(), config.getStreamKeySerde(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig1.KEY_SERDE(), + doTestSamzaPropertyDoesNotExist(StreamConfigJava.KEY_SERDE, (config, systemStream) -> assertEquals(Option.empty(), config.getStreamKeySerde(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig1::getStreamKeySerde); + doTestSamzaPropertyInvalidConfig(StreamConfigJava::getStreamKeySerde); } @Test public void testGetResetOffset() { - doTestSamzaProperty(StreamConfig1.CONSUMER_RESET_OFFSET(), "true", + doTestSamzaProperty(StreamConfigJava.CONSUMER_RESET_OFFSET, "true", (config, systemStream) -> assertTrue(config.getResetOffset(systemStream))); - doTestSamzaProperty(StreamConfig1.CONSUMER_RESET_OFFSET(), "false", + doTestSamzaProperty(StreamConfigJava.CONSUMER_RESET_OFFSET, "false", (config, systemStream) -> assertFalse(config.getResetOffset(systemStream))); // if not true/false, then use false - doTestSamzaProperty(StreamConfig1.CONSUMER_RESET_OFFSET(), "unknown_value", + doTestSamzaProperty(StreamConfigJava.CONSUMER_RESET_OFFSET, "unknown_value", (config, systemStream) -> assertFalse(config.getResetOffset(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig1.CONSUMER_RESET_OFFSET(), + doTestSamzaPropertyDoesNotExist(StreamConfigJava.CONSUMER_RESET_OFFSET, (config, systemStream) -> assertFalse(config.getResetOffset(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig1::getResetOffset); + doTestSamzaPropertyInvalidConfig(StreamConfigJava::getResetOffset); } @Test public void testIsResetOffsetConfigured() { - doTestSamzaProperty(StreamConfig1.CONSUMER_RESET_OFFSET(), "true", + doTestSamzaProperty(StreamConfigJava.CONSUMER_RESET_OFFSET, "true", (config, systemStream) -> assertTrue(config.isResetOffsetConfigured(systemStream))); - doTestSamzaProperty(StreamConfig1.CONSUMER_RESET_OFFSET(), "false", + doTestSamzaProperty(StreamConfigJava.CONSUMER_RESET_OFFSET, "false", (config, systemStream) -> assertTrue(config.isResetOffsetConfigured(systemStream))); // if not true/false, then use false - doTestSamzaProperty(StreamConfig1.CONSUMER_RESET_OFFSET(), "unknown_value", + doTestSamzaProperty(StreamConfigJava.CONSUMER_RESET_OFFSET, "unknown_value", (config, systemStream) -> assertTrue(config.isResetOffsetConfigured(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig1.CONSUMER_RESET_OFFSET(), + doTestSamzaPropertyDoesNotExist(StreamConfigJava.CONSUMER_RESET_OFFSET, (config, systemStream) -> assertFalse(config.isResetOffsetConfigured(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig1::isResetOffsetConfigured); + doTestSamzaPropertyInvalidConfig(StreamConfigJava::isResetOffsetConfigured); } @Test public void testGetDefaultStreamOffset() { String value = "my_offset_default"; - doTestSamzaProperty(StreamConfig1.CONSUMER_OFFSET_DEFAULT(), value, + doTestSamzaProperty(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, value, (config, systemStream) -> assertEquals(Option.apply(value), config.getDefaultStreamOffset(systemStream))); - doTestSamzaProperty(StreamConfig1.CONSUMER_OFFSET_DEFAULT(), "", + doTestSamzaProperty(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, "", (config, systemStream) -> assertEquals(Option.apply(""), config.getDefaultStreamOffset(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig1.CONSUMER_OFFSET_DEFAULT(), + doTestSamzaPropertyDoesNotExist(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, (config, systemStream) -> assertEquals(Option.empty(), - new StreamConfig1(config).getDefaultStreamOffset(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig1::getDefaultStreamOffset); + new StreamConfigJava(config).getDefaultStreamOffset(systemStream))); + doTestSamzaPropertyInvalidConfig(StreamConfigJava::getDefaultStreamOffset); } @Test public void testIsDefaultStreamOffsetConfigured() { String value = "my_offset_default"; - doTestSamzaProperty(StreamConfig1.CONSUMER_OFFSET_DEFAULT(), value, + doTestSamzaProperty(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, value, (config, systemStream) -> assertTrue(config.isDefaultStreamOffsetConfigured(systemStream))); - doTestSamzaProperty(StreamConfig1.CONSUMER_OFFSET_DEFAULT(), "", + doTestSamzaProperty(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, "", (config, systemStream) -> assertTrue(config.isDefaultStreamOffsetConfigured(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig1.CONSUMER_OFFSET_DEFAULT(), + doTestSamzaPropertyDoesNotExist(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, (config, systemStream) -> assertFalse(config.isDefaultStreamOffsetConfigured(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig1::isDefaultStreamOffsetConfigured); + doTestSamzaPropertyInvalidConfig(StreamConfigJava::isDefaultStreamOffsetConfigured); } @Test public void testGetBootstrapEnabled() { - doTestSamzaProperty(StreamConfig1.BOOTSTRAP(), "true", + doTestSamzaProperty(StreamConfigJava.BOOTSTRAP, "true", (config, systemStream) -> assertTrue(config.getBootstrapEnabled(systemStream))); - doTestSamzaProperty(StreamConfig1.BOOTSTRAP(), "false", + doTestSamzaProperty(StreamConfigJava.BOOTSTRAP, "false", (config, systemStream) -> assertFalse(config.getBootstrapEnabled(systemStream))); // if not true/false, then use false - doTestSamzaProperty(StreamConfig1.BOOTSTRAP(), "unknown_value", + doTestSamzaProperty(StreamConfigJava.BOOTSTRAP, "unknown_value", (config, systemStream) -> assertFalse(config.getBootstrapEnabled(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig1.BOOTSTRAP(), + doTestSamzaPropertyDoesNotExist(StreamConfigJava.BOOTSTRAP, (config, systemStream) -> assertFalse(config.getBootstrapEnabled(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig1::getBootstrapEnabled); + doTestSamzaPropertyInvalidConfig(StreamConfigJava::getBootstrapEnabled); } @Test public void testGetBroadcastEnabled() { - doTestSamzaProperty(StreamConfig1.BROADCAST(), "true", + doTestSamzaProperty(StreamConfigJava.BROADCAST, "true", (config, systemStream) -> assertTrue(config.getBroadcastEnabled(systemStream))); - doTestSamzaProperty(StreamConfig1.BROADCAST(), "false", + doTestSamzaProperty(StreamConfigJava.BROADCAST, "false", (config, systemStream) -> assertFalse(config.getBroadcastEnabled(systemStream))); // if not true/false, then use false - doTestSamzaProperty(StreamConfig1.BROADCAST(), "unknown_value", + doTestSamzaProperty(StreamConfigJava.BROADCAST, "unknown_value", (config, systemStream) -> assertFalse(config.getBroadcastEnabled(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig1.BROADCAST(), + doTestSamzaPropertyDoesNotExist(StreamConfigJava.BROADCAST, (config, systemStream) -> assertFalse(config.getBroadcastEnabled(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig1::getBroadcastEnabled); + doTestSamzaPropertyInvalidConfig(StreamConfigJava::getBroadcastEnabled); } @Test public void testGetPriority() { - doTestSamzaProperty(StreamConfig1.PRIORITY(), "0", + doTestSamzaProperty(StreamConfigJava.PRIORITY, "0", (config, systemStream) -> assertEquals(0, config.getPriority(systemStream))); - doTestSamzaProperty(StreamConfig1.PRIORITY(), "100", + doTestSamzaProperty(StreamConfigJava.PRIORITY, "100", (config, systemStream) -> assertEquals(100, config.getPriority(systemStream))); - doTestSamzaProperty(StreamConfig1.PRIORITY(), "-1", + doTestSamzaProperty(StreamConfigJava.PRIORITY, "-1", (config, systemStream) -> assertEquals(-1, config.getPriority(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfig1.PRIORITY(), + doTestSamzaPropertyDoesNotExist(StreamConfigJava.PRIORITY, (config, systemStream) -> assertEquals(-1, config.getPriority(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfig1::getPriority); + doTestSamzaPropertyInvalidConfig(StreamConfigJava::getPriority); } @Test public void testGetSerdeStreams() { assertEquals(Collections.emptySet(), - JavaConverters.setAsJavaSetConverter(new StreamConfig1(new MapConfig()).getSerdeStreams(SYSTEM)).asJava()); + new StreamConfigJava(new MapConfig()).getSerdeStreams(SYSTEM)); // not key/msg serde property for "streams." - StreamConfig1 streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); - assertEquals(Collections.emptySet(), - JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); + StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); + assertEquals(Collections.emptySet(), streamConfig.getSerdeStreams(SYSTEM)); // not matching system for "streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.KEY_SERDE(), UNUSED_VALUE, - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), "otherSystem"))); - assertEquals(Collections.emptySet(), - JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.KEY_SERDE, UNUSED_VALUE, + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), "otherSystem"))); + assertEquals(Collections.emptySet(), streamConfig.getSerdeStreams(SYSTEM)); // not key/msg serde property for "systems..streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); - assertEquals(Collections.emptySet(), - JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); + assertEquals(Collections.emptySet(),streamConfig.getSerdeStreams(SYSTEM)); // not matching system for "systems..streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), "otherSystem", STREAM_ID) + StreamConfig1.KEY_SERDE(), + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, "otherSystem", STREAM_ID) + StreamConfigJava.KEY_SERDE, UNUSED_VALUE))); - assertEquals(Collections.emptySet(), - JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); + assertEquals(Collections.emptySet(), streamConfig.getSerdeStreams(SYSTEM)); // not matching system for "streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.KEY_SERDE(), UNUSED_VALUE, - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), "otherSystem"))); - assertEquals(Collections.emptySet(), - JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.KEY_SERDE, UNUSED_VALUE, + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), "otherSystem"))); + assertEquals(Collections.emptySet(),streamConfig.getSerdeStreams(SYSTEM)); String serdeValue = "my.serde.class"; // key serde for "streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.KEY_SERDE(), serdeValue))); - assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), - JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.KEY_SERDE, serdeValue))); + assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), streamConfig.getSerdeStreams(SYSTEM)); // msg serde for "streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.MSG_SERDE(), serdeValue))); - assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), - JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.MSG_SERDE, serdeValue))); + assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), streamConfig.getSerdeStreams(SYSTEM)); // serde for "streams." with physical stream name mapping - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM, - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.KEY_SERDE(), serdeValue))); - assertEquals(Collections.singleton(new SystemStream(SYSTEM, PHYSICAL_STREAM)), - JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM, + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.KEY_SERDE, serdeValue))); + assertEquals(Collections.singleton(new SystemStream(SYSTEM, PHYSICAL_STREAM)), streamConfig.getSerdeStreams(SYSTEM)); // key serde for "systems..streams." - streamConfig = new StreamConfig1(new MapConfig( - ImmutableMap.of(String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + StreamConfig1.KEY_SERDE(), + streamConfig = new StreamConfigJava(new MapConfig( + ImmutableMap.of(String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + StreamConfigJava.KEY_SERDE, serdeValue))); - assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), - JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); + assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), streamConfig.getSerdeStreams(SYSTEM)); // msg serde for "systems..streams." - streamConfig = new StreamConfig1(new MapConfig( - ImmutableMap.of(String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + StreamConfig1.MSG_SERDE(), + streamConfig = new StreamConfigJava(new MapConfig( + ImmutableMap.of(String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + StreamConfigJava.MSG_SERDE, serdeValue))); - assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), - JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); + assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), streamConfig.getSerdeStreams(SYSTEM)); // merge several different ways of providing serdes String streamIdWithPhysicalName = "streamIdWithPhysicalName"; - streamConfig = new StreamConfig1(new MapConfig(new ImmutableMap.Builder() + streamConfig = new StreamConfigJava(new MapConfig(new ImmutableMap.Builder() // need to map the stream ids to the system .put(JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM) // key and msg serde for "streams." - .put(String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.KEY_SERDE(), serdeValue) - .put(String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + StreamConfig1.MSG_SERDE(), serdeValue) + .put(String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.KEY_SERDE, serdeValue) + .put(String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.MSG_SERDE, serdeValue) // key serde for "streams." with physical stream name mapping - .put(String.format(StreamConfig1.STREAM_ID_PREFIX(), streamIdWithPhysicalName) + StreamConfig1.KEY_SERDE(), + .put(String.format(StreamConfigJava.STREAM_ID_PREFIX, streamIdWithPhysicalName) + StreamConfigJava.KEY_SERDE, serdeValue) - .put(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), streamIdWithPhysicalName), PHYSICAL_STREAM) + .put(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, streamIdWithPhysicalName), PHYSICAL_STREAM) // key serde for "systems..streams." - .put(String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, OTHER_STREAM_ID) + StreamConfig1.KEY_SERDE(), + .put(String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, OTHER_STREAM_ID) + StreamConfigJava.KEY_SERDE, serdeValue) .build())); assertEquals(Sets.newHashSet(new SystemStream(SYSTEM, STREAM_ID), new SystemStream(SYSTEM, PHYSICAL_STREAM), - new SystemStream(SYSTEM, OTHER_STREAM_ID)), - JavaConverters.setAsJavaSetConverter(streamConfig.getSerdeStreams(SYSTEM)).asJava()); + new SystemStream(SYSTEM, OTHER_STREAM_ID)), streamConfig.getSerdeStreams(SYSTEM)); } @Test public void testGetStreamProperties() { - assertEquals(new MapConfig(), new StreamConfig1(new MapConfig()).getStreamProperties(STREAM_ID)); + assertEquals(new MapConfig(), new StreamConfigJava(new MapConfig()).getStreamProperties(STREAM_ID)); String propertyName = "stream.property.name"; // BEGIN: tests in which properties cannot be found in the config // not matching stream id for "streams." - StreamConfig1 streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), OTHER_STREAM_ID) + propertyName, UNUSED_VALUE))); + StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, OTHER_STREAM_ID) + propertyName, UNUSED_VALUE))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // not matching stream id for "systems..streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, OTHER_STREAM_ID) + propertyName, UNUSED_VALUE, - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), OTHER_STREAM_ID), SYSTEM))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, OTHER_STREAM_ID) + propertyName, UNUSED_VALUE, + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, OTHER_STREAM_ID), SYSTEM))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // no system mapping when using "systems..streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // ignore property with "samza" prefix for "streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // ignore property with "samza" prefix for "streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE, - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE, + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // should not map physical name back to stream id if physical name is passed as stream id - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(PHYSICAL_STREAM)); // BEGIN: tests in which properties can be found in the config @@ -306,65 +295,65 @@ public void testGetStreamProperties() { String propertyValue = "value"; // "streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, propertyValue))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + propertyName, propertyValue))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // "systems..streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, propertyValue, - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, propertyValue, + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // "systems..default.stream." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, propertyValue, - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // use physical name mapping for "systems..streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, PHYSICAL_STREAM) + propertyName, propertyValue, + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, PHYSICAL_STREAM) + propertyName, propertyValue, // should not use stream id since there is physical stream - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // "streams." should override "systems..streams." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, propertyValue, + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + propertyName, propertyValue, // should not use "systems..streams." since there is a "streams." config - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // "systems..streams." should override "systems..default.stream." - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, propertyValue, + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, propertyValue, // should not use "systems..default.stream." since there is a "systems..streams." String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, UNUSED_VALUE, - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // merge multiple ways of specifying configs - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( // "streams." - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + "from.stream.id.property", "fromStreamIdValue", + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + "from.stream.id.property", "fromStreamIdValue", // second "streams." property - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + "from.stream.id.other.property", + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + "from.stream.id.other.property", "fromStreamIdOtherValue", // "systems..streams." - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + "from.system.stream.property", + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + "from.system.stream.property", "fromSystemStreamValue", // need to map the stream id to a system - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))); + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of( "from.stream.id.property", "fromStreamIdValue", "from.stream.id.other.property", "fromStreamIdOtherValue", @@ -374,133 +363,133 @@ public void testGetStreamProperties() { @Test public void testGetSystem() { - assertNull(new StreamConfig1(new MapConfig()).getSystem(STREAM_ID)); + assertNull(new StreamConfigJava(new MapConfig()).getSystem(STREAM_ID)); // system is specified directly - StreamConfig1 streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, JobConfig.JOB_DEFAULT_SYSTEM, "otherSystem", - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), OTHER_STREAM_ID), "otherSystem"))); + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, OTHER_STREAM_ID), "otherSystem"))); assertEquals(SYSTEM, streamConfig.getSystem(STREAM_ID)); // fall back to job default system - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), OTHER_STREAM_ID), "otherSystem"))); + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, OTHER_STREAM_ID), "otherSystem"))); assertEquals(SYSTEM, streamConfig.getSystem(STREAM_ID)); } @Test public void testGetPhysicalName() { - assertEquals(STREAM_ID, new StreamConfig1(new MapConfig()).getPhysicalName(STREAM_ID)); + assertEquals(STREAM_ID, new StreamConfigJava(new MapConfig()).getPhysicalName(STREAM_ID)); // ignore mapping for other stream ids - StreamConfig1 streamConfig = new StreamConfig1(new MapConfig( - ImmutableMap.of(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), OTHER_STREAM_ID), PHYSICAL_STREAM))); + StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig( + ImmutableMap.of(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, OTHER_STREAM_ID), PHYSICAL_STREAM))); assertEquals(STREAM_ID, streamConfig.getPhysicalName(STREAM_ID)); - streamConfig = new StreamConfig1(new MapConfig( - ImmutableMap.of(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfigJava(new MapConfig( + ImmutableMap.of(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); assertEquals(PHYSICAL_STREAM, streamConfig.getPhysicalName(STREAM_ID)); } @Test public void testGetIsIntermediateStream() { - assertFalse(new StreamConfig1(new MapConfig()).getIsIntermediateStream(STREAM_ID)); + assertFalse(new StreamConfigJava(new MapConfig()).getIsIntermediateStream(STREAM_ID)); // ignore mapping for other stream ids - StreamConfig1 streamConfig = new StreamConfig1(new MapConfig( - ImmutableMap.of(String.format(StreamConfig1.IS_INTERMEDIATE_FOR_STREAM_ID(), OTHER_STREAM_ID), "true"))); + StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig( + ImmutableMap.of(String.format(StreamConfigJava.IS_INTERMEDIATE_FOR_STREAM_ID, OTHER_STREAM_ID), "true"))); assertFalse(streamConfig.getIsIntermediateStream(STREAM_ID)); // do not use stream id property if physical name is passed as input - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.IS_INTERMEDIATE_FOR_STREAM_ID(), STREAM_ID), "true", - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.IS_INTERMEDIATE_FOR_STREAM_ID, STREAM_ID), "true", + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); assertFalse(streamConfig.getIsIntermediateStream(PHYSICAL_STREAM)); - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.IS_INTERMEDIATE_FOR_STREAM_ID(), STREAM_ID), "true"))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.IS_INTERMEDIATE_FOR_STREAM_ID, STREAM_ID), "true"))); assertTrue(streamConfig.getIsIntermediateStream(STREAM_ID)); - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.IS_INTERMEDIATE_FOR_STREAM_ID(), STREAM_ID), "false"))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.IS_INTERMEDIATE_FOR_STREAM_ID, STREAM_ID), "false"))); assertFalse(streamConfig.getIsIntermediateStream(STREAM_ID)); } @Test public void testGetDeleteCommittedMessages() { - assertFalse(new StreamConfig1(new MapConfig()).getDeleteCommittedMessages(STREAM_ID)); + assertFalse(new StreamConfigJava(new MapConfig()).getDeleteCommittedMessages(STREAM_ID)); // ignore mapping for other stream ids - StreamConfig1 streamConfig = new StreamConfig1(new MapConfig( - ImmutableMap.of(String.format(StreamConfig1.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), OTHER_STREAM_ID), + StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig( + ImmutableMap.of(String.format(StreamConfigJava.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, OTHER_STREAM_ID), "true"))); assertFalse(streamConfig.getDeleteCommittedMessages(STREAM_ID)); // do not use stream id property if physical name is passed as input - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), STREAM_ID), "true", - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, STREAM_ID), "true", + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); assertFalse(streamConfig.getDeleteCommittedMessages(PHYSICAL_STREAM)); - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), STREAM_ID), "true"))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, STREAM_ID), "true"))); assertTrue(streamConfig.getDeleteCommittedMessages(STREAM_ID)); - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), STREAM_ID), "false"))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, STREAM_ID), "false"))); assertFalse(streamConfig.getDeleteCommittedMessages(STREAM_ID)); } @Test public void testGetIsBounded() { - assertFalse(new StreamConfig1(new MapConfig()).getIsBounded(STREAM_ID)); + assertFalse(new StreamConfigJava(new MapConfig()).getIsBounded(STREAM_ID)); // ignore mapping for other stream ids - StreamConfig1 streamConfig = new StreamConfig1(new MapConfig( - ImmutableMap.of(String.format(StreamConfig1.IS_BOUNDED_FOR_STREAM_ID(), OTHER_STREAM_ID), + StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig( + ImmutableMap.of(String.format(StreamConfigJava.IS_BOUNDED_FOR_STREAM_ID, OTHER_STREAM_ID), "true"))); assertFalse(streamConfig.getIsBounded(STREAM_ID)); // do not use stream id property if physical name is passed as input - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.IS_BOUNDED_FOR_STREAM_ID(), STREAM_ID), "true", - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.IS_BOUNDED_FOR_STREAM_ID, STREAM_ID), "true", + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); assertFalse(streamConfig.getIsBounded(PHYSICAL_STREAM)); - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.IS_BOUNDED_FOR_STREAM_ID(), STREAM_ID), "true"))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.IS_BOUNDED_FOR_STREAM_ID, STREAM_ID), "true"))); assertTrue(streamConfig.getIsBounded(STREAM_ID)); - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.IS_BOUNDED_FOR_STREAM_ID(), STREAM_ID), "false"))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.IS_BOUNDED_FOR_STREAM_ID, STREAM_ID), "false"))); assertFalse(streamConfig.getIsBounded(STREAM_ID)); } @Test public void testGetStreamIds() { assertEquals(ImmutableList.of(), ImmutableList.copyOf( - JavaConverters.asJavaIterableConverter(new StreamConfig1(new MapConfig()).getStreamIds()).asJava())); + new StreamConfigJava(new MapConfig()).getStreamIds())); - StreamConfig1 streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + ".property", "value"))); + StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + ".property", "value"))); assertEquals(ImmutableList.of(STREAM_ID), - ImmutableList.copyOf(JavaConverters.asJavaIterableConverter(streamConfig.getStreamIds()).asJava())); + ImmutableList.copyOf(streamConfig.getStreamIds())); - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + ".property.subProperty", "value"))); + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + ".property.subProperty", "value"))); assertEquals(ImmutableList.of(STREAM_ID), - ImmutableList.copyOf(JavaConverters.asJavaIterableConverter(streamConfig.getStreamIds()).asJava())); - - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + ".property0", "value", - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + ".property1", "value", - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + ".property.subProperty0", "value", - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + ".property.subProperty1", "value", - String.format(StreamConfig1.STREAM_ID_PREFIX(), OTHER_STREAM_ID) + ".property", "value"))); + ImmutableList.copyOf(streamConfig.getStreamIds())); + + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + ".property0", "value", + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + ".property1", "value", + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + ".property.subProperty0", "value", + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + ".property.subProperty1", "value", + String.format(StreamConfigJava.STREAM_ID_PREFIX, OTHER_STREAM_ID) + ".property", "value"))); assertEquals(ImmutableList.of(STREAM_ID, OTHER_STREAM_ID), - ImmutableList.copyOf(JavaConverters.asJavaIterableConverter(streamConfig.getStreamIds()).asJava())); + ImmutableList.copyOf(streamConfig.getStreamIds())); } private static void doTestSamzaProperty(String propertyName, String propertyValue, SamzaPropertyAssertion assertion) { @@ -517,63 +506,63 @@ private static void doTestSamzaProperty(String propertyName, String propertyValu */ private static void doTestSamzaPropertyAccess(String propertyName, String value, SamzaPropertyAssertion assertion) { // streams.. - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + propertyName, value, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM))), SYSTEM_STREAM); // systems..streams.. where stream id has no specified physical stream - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, // specify the system for the stream id - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))), + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))), SYSTEM_STREAM); // systems..streams.. where stream is the streamId, system is from job default system - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, // use job default system to get the system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM))), SYSTEM_STREAM); // systems..default.stream. where stream id has no specified physical stream - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // specify the system for the stream id - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM))), + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))), SYSTEM_STREAM); // systems..streams.. where no system mapping (fall back to SystemStream) - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, // map the stream id to the physical stream - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); // systems..default.stream. where no system mapping (fall back to SystemStream) - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // map the stream id to the physical stream - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); // systems..streams.. where stream id has a physical name but property is from stream id - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, // use job default system to get the system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); // systems..default.stream. where stream id has a physical name but property is from stream id - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // use job default system to get the system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); } @@ -583,39 +572,39 @@ private static void doTestSamzaPropertyAccess(String propertyName, String value, private static void doTestSamzaPropertyAccessWithPhysicalStream(String propertyName, String value, SamzaPropertyAssertion assertion) { // streams.. - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + propertyName, value, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); // systems..streams.. with a specific system for the stream id - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, PHYSICAL_STREAM) + propertyName, value, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, PHYSICAL_STREAM) + propertyName, value, // specify the system for the stream id - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); // systems..streams.. with system coming from job default system - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, PHYSICAL_STREAM) + propertyName, value, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, PHYSICAL_STREAM) + propertyName, value, // use job default system to get the system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); // systems..default.stream. - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // specify the system for the stream id - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); } @@ -626,19 +615,19 @@ private static void doTestSamzaPropertyAccessWithPhysicalStream(String propertyN */ private static void doTestSamzaPropertyPriority(String propertyName, String value, SamzaPropertyAssertion assertion) { // streams.. vs. systems..streams.. - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + propertyName, value, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // this config should not be used - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE))), + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE))), SYSTEM_STREAM); // systems..streams.. vs. systems..default.stream. - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, // specify the system for the stream id - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, // this config should not be used String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, UNUSED_VALUE))), SYSTEM_STREAM); @@ -653,8 +642,8 @@ private static void doTestSamzaPropertyPriority(String propertyName, String valu * systems..streams.. without a system mapping in the config vs. * systems..default.stream. without a system mapping in the config */ - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, // this config should not be used String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, UNUSED_VALUE))), SYSTEM_STREAM); @@ -666,19 +655,19 @@ private static void doTestSamzaPropertyPriority(String propertyName, String valu */ private static void doTestSamzaPropertyMultipleStreams(String propertyName, String value, SamzaPropertyAssertion assertion) { // systems..default.stream. where stream id has no specified physical stream - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // specify the systems for the streams - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID), SYSTEM, - String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), OTHER_STREAM_ID), SYSTEM))), + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, OTHER_STREAM_ID), SYSTEM))), SYSTEM_STREAM); } private static void doTestSamzaPropertyInvalidConfig(SamzaPropertyLookup lookup) { // configure physical stream and have mapping from stream id to physical stream - StreamConfig1 streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM, - String.format(StreamConfig1.STREAM_ID_PREFIX(), PHYSICAL_STREAM) + ".property", "value", + StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM, + String.format(StreamConfigJava.STREAM_ID_PREFIX, PHYSICAL_STREAM) + ".property", "value", JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM))); try { lookup.doLookup(streamConfig, SYSTEM_STREAM_PHYSICAL); @@ -688,9 +677,9 @@ private static void doTestSamzaPropertyInvalidConfig(SamzaPropertyLookup lookup) } // two separate stream ids map to same physical stream - streamConfig = new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM, - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), OTHER_STREAM_ID), PHYSICAL_STREAM, + streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM, + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, OTHER_STREAM_ID), PHYSICAL_STREAM, JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM))); try { lookup.doLookup(streamConfig, SYSTEM_STREAM_PHYSICAL); @@ -701,42 +690,42 @@ private static void doTestSamzaPropertyInvalidConfig(SamzaPropertyLookup lookup) } private static void doTestSamzaPropertyDoesNotExist(String propertyName, SamzaPropertyAssertion assertion) { - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( // just put in some value which will be ignored - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))), + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))), SYSTEM_STREAM); /* * Won't use streams.. if streamId is mapped to a physical stream */ - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_ID_PREFIX(), STREAM_ID) + propertyName, UNUSED_VALUE, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + propertyName, UNUSED_VALUE, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); /* * Won't use systems..streams.. if stream is mapped to a physical stream and the property * is only specified using the physical stream */ - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, PHYSICAL_STREAM) + propertyName, UNUSED_VALUE, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, PHYSICAL_STREAM) + propertyName, UNUSED_VALUE, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); /* * Won't use systems..streams.. if stream is mapped to a physical stream and there is no * system mapping for the stream id */ - assertion.doAssertion(new StreamConfig1(new MapConfig(ImmutableMap.of( - String.format(StreamConfig1.STREAM_PREFIX(), SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, + assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, // map the stream id to the physical stream - String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); } @@ -745,7 +734,7 @@ private static void doTestSamzaPropertyDoesNotExist(String propertyName, SamzaPr */ @FunctionalInterface private interface SamzaPropertyAssertion { - void doAssertion(StreamConfig1 streamConfig, SystemStream systemStream); + void doAssertion(StreamConfigJava streamConfig, SystemStream systemStream); } /** @@ -753,6 +742,6 @@ private interface SamzaPropertyAssertion { */ @FunctionalInterface private interface SamzaPropertyLookup { - void doLookup(StreamConfig1 streamConfig, SystemStream systemStream); + void doLookup(StreamConfigJava streamConfig, SystemStream systemStream); } } \ No newline at end of file diff --git a/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java b/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java index c1d6223342..12a65a50b8 100644 --- a/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java +++ b/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java @@ -26,6 +26,7 @@ import org.apache.samza.system.SystemFactory; import org.apache.samza.system.SystemProducer; import org.junit.Test; + import java.util.HashMap; import java.util.Map; @@ -129,7 +130,7 @@ public void testGetDefaultStreamProperties() { new MapConfig(ImmutableMap.of(system1ConfigKey, system1ConfigValue, system2ConfigKey, system2ConfigValue)); SystemConfig systemConfig = new SystemConfig(config); assertEquals(expected, systemConfig.getDefaultStreamProperties(MOCK_SYSTEM_NAME1)); - assertEquals(new MapConfig(), systemConfig.getDefaultStreamProperties(MOCK_SYSTEM_NAME2)); + assertEquals(new MapConfig, systemConfig.getDefaultStreamProperties(MOCK_SYSTEM_NAME2)); } @Test @@ -157,13 +158,13 @@ public void testGetSystemKeySerde() { // value specified explicitly Config config = - new MapConfig(ImmutableMap.of(defaultStreamPrefixSystem1 + StreamConfig1.KEY_SERDE(), system1KeySerde)); + new MapConfig(ImmutableMap.of(defaultStreamPrefixSystem1 + StreamConfigJava.KEY_SERDE, system1KeySerde)); SystemConfig systemConfig = new SystemConfig(config); assertEquals(system1KeySerde, systemConfig.getSystemKeySerde(MOCK_SYSTEM_NAME1).get()); // default stream property is unspecified, try fall back config key config = new MapConfig( - ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig1.KEY_SERDE(), + ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfigJava.KEY_SERDE, system1KeySerde)); systemConfig = new SystemConfig(config); assertEquals(system1KeySerde, systemConfig.getSystemKeySerde(MOCK_SYSTEM_NAME1).get()); @@ -171,15 +172,15 @@ public void testGetSystemKeySerde() { // default stream property is empty string, try fall back config key config = new MapConfig(ImmutableMap.of( // default stream property is empty - defaultStreamPrefixSystem1 + StreamConfig1.KEY_SERDE(), "", + defaultStreamPrefixSystem1 + StreamConfigJava.KEY_SERDE, "", // fall back entry - String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig1.KEY_SERDE(), system1KeySerde)); + String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfigJava.KEY_SERDE, system1KeySerde)); systemConfig = new SystemConfig(config); assertEquals(system1KeySerde, systemConfig.getSystemKeySerde(MOCK_SYSTEM_NAME1).get()); // default stream property is unspecified, fall back is also empty config = new MapConfig( - ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig1.KEY_SERDE(), + ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfigJava.KEY_SERDE, "")); systemConfig = new SystemConfig(config); assertFalse(systemConfig.getSystemKeySerde(MOCK_SYSTEM_NAME1).isPresent()); @@ -197,13 +198,13 @@ public void testGetSystemMsgSerde() { // value specified explicitly Config config = - new MapConfig(ImmutableMap.of(defaultStreamPrefixSystem1 + StreamConfig1.MSG_SERDE(), system1MsgSerde)); + new MapConfig(ImmutableMap.of(defaultStreamPrefixSystem1 + StreamConfigJava.MSG_SERDE, system1MsgSerde)); SystemConfig systemConfig = new SystemConfig(config); assertEquals(system1MsgSerde, systemConfig.getSystemMsgSerde(MOCK_SYSTEM_NAME1).get()); // default stream property is unspecified, try fall back config msg config = new MapConfig( - ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig1.MSG_SERDE(), + ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfigJava.MSG_SERDE, system1MsgSerde)); systemConfig = new SystemConfig(config); assertEquals(system1MsgSerde, systemConfig.getSystemMsgSerde(MOCK_SYSTEM_NAME1).get()); @@ -211,15 +212,15 @@ public void testGetSystemMsgSerde() { // default stream property is empty string, try fall back config msg config = new MapConfig(ImmutableMap.of( // default stream property is empty - defaultStreamPrefixSystem1 + StreamConfig1.MSG_SERDE(), "", + defaultStreamPrefixSystem1 + StreamConfigJava.MSG_SERDE, "", // fall back entry - String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig1.MSG_SERDE(), system1MsgSerde)); + String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfigJava.MSG_SERDE, system1MsgSerde)); systemConfig = new SystemConfig(config); assertEquals(system1MsgSerde, systemConfig.getSystemMsgSerde(MOCK_SYSTEM_NAME1).get()); // default stream property is unspecified, fall back is also empty config = new MapConfig( - ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig1.MSG_SERDE(), + ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfigJava.MSG_SERDE, "")); systemConfig = new SystemConfig(config); assertFalse(systemConfig.getSystemMsgSerde(MOCK_SYSTEM_NAME1).isPresent()); diff --git a/samza-core/src/test/java/org/apache/samza/execution/TestStreamEdge.java b/samza-core/src/test/java/org/apache/samza/execution/TestStreamEdge.java index 1be25862d4..366e017207 100644 --- a/samza-core/src/test/java/org/apache/samza/execution/TestStreamEdge.java +++ b/samza-core/src/test/java/org/apache/samza/execution/TestStreamEdge.java @@ -22,7 +22,7 @@ import org.apache.samza.config.ApplicationConfig; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.system.StreamSpec; import org.junit.Test; @@ -67,7 +67,7 @@ public void testGenerateConfig() { StreamSpec spec = new StreamSpec("stream-1", "physical-stream-1", "system-1", Collections.singletonMap("property1", "haha")); StreamEdge edge = new StreamEdge(spec, false, false, new MapConfig()); Config config = edge.generateConfig(); - StreamConfig1 streamConfig = new StreamConfig1(config); + StreamConfigJava streamConfig = new StreamConfigJava(config); assertEquals(streamConfig.getSystem(spec.getId()), "system-1"); assertEquals(streamConfig.getPhysicalName(spec.getId()), "physical-stream-1"); assertEquals(streamConfig.getIsIntermediateStream(spec.getId()), false); @@ -77,12 +77,12 @@ public void testGenerateConfig() { spec = new StreamSpec("stream-1", "physical-stream-1", "system-1", Collections.singletonMap("property1", "haha")); edge = new StreamEdge(spec, false, false, new MapConfig()); config = edge.generateConfig(); - streamConfig = new StreamConfig1(config); + streamConfig = new StreamConfigJava(config); // intermediate stream edge = new StreamEdge(spec, true, false, new MapConfig()); config = edge.generateConfig(); - streamConfig = new StreamConfig1(config); + streamConfig = new StreamConfigJava(config); assertEquals(streamConfig.getIsIntermediateStream(spec.getId()), true); assertEquals(streamConfig.getPriority(spec.toSystemStream()), Integer.MAX_VALUE); } diff --git a/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImplGraph.java b/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImplGraph.java index b908a3c8f6..99ec93b9f9 100644 --- a/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImplGraph.java +++ b/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImplGraph.java @@ -26,7 +26,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.container.TaskName; import org.apache.samza.context.Context; import org.apache.samza.context.MockContext; @@ -504,7 +504,7 @@ public void testGetOutputToInputStreams() { }, config); Multimap outputToInput = - OperatorImplGraph.getIntermediateToInputStreamsMap(graphSpec.getOperatorSpecGraph(), new StreamConfig1(config)); + OperatorImplGraph.getIntermediateToInputStreamsMap(graphSpec.getOperatorSpecGraph(), new StreamConfigJava(config)); Collection inputs = outputToInput.get(new SystemStream(intSystem, intStreamId2)); assertEquals(inputs.size(), 2); assertTrue(inputs.contains(new SystemStream(inputSystem, inputStreamId1))); diff --git a/samza-core/src/test/java/org/apache/samza/util/TestStreamUtil.java b/samza-core/src/test/java/org/apache/samza/util/TestStreamUtil.java index 4d3d876fb9..9573ad1d89 100644 --- a/samza-core/src/test/java/org/apache/samza/util/TestStreamUtil.java +++ b/samza-core/src/test/java/org/apache/samza/util/TestStreamUtil.java @@ -18,15 +18,16 @@ */ package org.apache.samza.util; -import java.util.HashMap; -import java.util.Map; import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.system.StreamSpec; import org.junit.Test; +import java.util.HashMap; +import java.util.Map; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -49,10 +50,10 @@ public class TestStreamUtil { @Test public void testGetStreamWithPhysicalNameInConfig() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig1.SYSTEM(), TEST_SYSTEM); + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfigJava.SYSTEM, TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); assertEquals(TEST_PHYSICAL_NAME, spec.getPhysicalName()); } @@ -62,9 +63,9 @@ public void testGetStreamWithPhysicalNameInConfig() { @Test public void testGetStreamWithoutPhysicalNameInConfig() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.SYSTEM(), TEST_SYSTEM); + StreamConfigJava.SYSTEM, TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); assertEquals(STREAM_ID, spec.getPhysicalName()); } @@ -73,10 +74,10 @@ public void testGetStreamWithoutPhysicalNameInConfig() { @Test public void testGetStreamWithSystemAtStreamScopeInConfig() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig1.SYSTEM(), TEST_SYSTEM); + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfigJava.SYSTEM, TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); assertEquals(TEST_SYSTEM, spec.getSystemName()); } @@ -85,10 +86,10 @@ public void testGetStreamWithSystemAtStreamScopeInConfig() { @Test public void testGetStreamWithSystemAtDefaultScopeInConfig() { Config config = addConfigs(buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME), + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME), JobConfig.JOB_DEFAULT_SYSTEM, TEST_DEFAULT_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); assertEquals(TEST_DEFAULT_SYSTEM, spec.getSystemName()); } @@ -97,11 +98,11 @@ public void testGetStreamWithSystemAtDefaultScopeInConfig() { @Test public void testGetStreamWithSystemAtBothScopesInConfig() { Config config = addConfigs(buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig1.SYSTEM(), TEST_SYSTEM), + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfigJava.SYSTEM, TEST_SYSTEM), JobConfig.JOB_DEFAULT_SYSTEM, TEST_DEFAULT_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); assertEquals(TEST_SYSTEM, spec.getSystemName()); } @@ -110,9 +111,9 @@ public void testGetStreamWithSystemAtBothScopesInConfig() { @Test(expected = IllegalArgumentException.class) public void testGetStreamWithOutSystemInConfig() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME); + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); assertEquals(TEST_SYSTEM, spec.getSystemName()); } @@ -121,13 +122,13 @@ public void testGetStreamWithOutSystemInConfig() { @Test public void testGetStreamPropertiesPassthrough() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig1.SYSTEM(), TEST_SYSTEM, + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfigJava.SYSTEM, TEST_SYSTEM, "systemProperty1", "systemValue1", "systemProperty2", "systemValue2", "systemProperty3", "systemValue3"); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); Map properties = spec.getConfig(); assertEquals(3, properties.size()); @@ -143,35 +144,35 @@ public void testGetStreamPropertiesPassthrough() { @Test public void testGetStreamSamzaPropertiesOmitted() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig1.SYSTEM(), TEST_SYSTEM, + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfigJava.SYSTEM, TEST_SYSTEM, "systemProperty1", "systemValue1", "systemProperty2", "systemValue2", "systemProperty3", "systemValue3"); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); Map properties = spec.getConfig(); assertEquals(3, properties.size()); - assertNull(properties.get(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID))); - assertNull(properties.get(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID))); - assertNull(spec.get(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID))); - assertNull(spec.get(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), STREAM_ID))); + assertNull(properties.get(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID))); + assertNull(properties.get(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID))); + assertNull(spec.get(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID))); + assertNull(spec.get(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID))); } @Test public void testStreamConfigOverrides() { final String sysStreamPrefix = String.format("systems.%s.streams.%s.", TEST_SYSTEM, TEST_PHYSICAL_NAME); Config config = addConfigs(buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig1.SYSTEM(), TEST_SYSTEM, + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfigJava.SYSTEM, TEST_SYSTEM, "systemProperty1", "systemValue1", "systemProperty2", "systemValue2", "systemProperty3", "systemValue3"), sysStreamPrefix + "systemProperty4", "systemValue4", sysStreamPrefix + "systemProperty2", "systemValue8"); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); Map properties = spec.getConfig(); assertEquals(4, properties.size()); @@ -183,14 +184,14 @@ public void testStreamConfigOverrides() { @Test public void testStreamConfigOverridesWithSystemDefaults() { Config config = addConfigs(buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig1.SYSTEM(), TEST_SYSTEM, + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfigJava.SYSTEM, TEST_SYSTEM, "segment.bytes", "5309"), String.format("systems.%s.default.stream.replication.factor", TEST_SYSTEM), "4", // System default property String.format("systems.%s.default.stream.segment.bytest", TEST_SYSTEM), "867" ); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); Map properties = spec.getConfig(); assertEquals(3, properties.size()); @@ -202,10 +203,10 @@ public void testStreamConfigOverridesWithSystemDefaults() { @Test public void testGetStreamPhysicalNameArgSimple() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME2, // This should be ignored because of the explicit arg - StreamConfig1.SYSTEM(), TEST_SYSTEM); + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME2, // This should be ignored because of the explicit arg + StreamConfigJava.SYSTEM, TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); assertEquals(STREAM_ID, spec.getId()); assertEquals(TEST_PHYSICAL_NAME2, spec.getPhysicalName()); @@ -216,10 +217,10 @@ public void testGetStreamPhysicalNameArgSimple() { @Test public void testGetStreamPhysicalNameArgSpecialCharacters() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME_SPECIAL_CHARS, - StreamConfig1.SYSTEM(), TEST_SYSTEM); + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME_SPECIAL_CHARS, + StreamConfigJava.SYSTEM, TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); assertEquals(TEST_PHYSICAL_NAME_SPECIAL_CHARS, spec.getPhysicalName()); } @@ -227,10 +228,10 @@ public void testGetStreamPhysicalNameArgSpecialCharacters() { @Test public void testGetStreamPhysicalNameArgNull() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), null, - StreamConfig1.SYSTEM(), TEST_SYSTEM); + StreamConfigJava.PHYSICAL_NAME, null, + StreamConfigJava.SYSTEM, TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); assertNull(spec.getPhysicalName()); } @@ -238,10 +239,10 @@ public void testGetStreamPhysicalNameArgNull() { @Test public void testGetStreamSystemNameArgValid() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, // This should be ignored because of the explicit arg - StreamConfig1.SYSTEM(), TEST_SYSTEM); // This too + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, // This should be ignored because of the explicit arg + StreamConfigJava.SYSTEM, TEST_SYSTEM); // This too - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); assertEquals(STREAM_ID, spec.getId()); assertEquals(TEST_PHYSICAL_NAME, spec.getPhysicalName()); @@ -252,57 +253,57 @@ public void testGetStreamSystemNameArgValid() { @Test(expected = IllegalArgumentException.class) public void testGetStreamSystemNameArgInvalid() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig1.SYSTEM(), TEST_SYSTEM_INVALID); + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfigJava.SYSTEM, TEST_SYSTEM_INVALID); - StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); } // Empty strings are NOT allowed for system name, because it's used as an identifier in the config. @Test(expected = IllegalArgumentException.class) public void testGetStreamSystemNameArgEmpty() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig1.SYSTEM(), ""); + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfigJava.SYSTEM, ""); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); } // Null is not allowed IllegalArgumentException system name. @Test(expected = IllegalArgumentException.class) public void testGetStreamSystemNameArgNull() { Config config = buildStreamConfig(STREAM_ID, - StreamConfig1.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, - StreamConfig1.SYSTEM(), null); + StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfigJava.SYSTEM, null); - StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig1(config)); + StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); } // Special characters are NOT allowed for streamId, because it's used as an identifier in the config. @Test(expected = IllegalArgumentException.class) public void testGetStreamStreamIdInvalid() { Config config = buildStreamConfig(STREAM_ID_INVALID, - StreamConfig1.SYSTEM(), TEST_SYSTEM); + StreamConfigJava.SYSTEM, TEST_SYSTEM); - StreamUtil.getStreamSpec(STREAM_ID_INVALID, new StreamConfig1(config)); + StreamUtil.getStreamSpec(STREAM_ID_INVALID, new StreamConfigJava(config)); } // Empty strings are NOT allowed for streamId, because it's used as an identifier in the config. @Test(expected = IllegalArgumentException.class) public void testGetStreamStreamIdEmpty() { Config config = buildStreamConfig("", - StreamConfig1.SYSTEM(), TEST_SYSTEM); + StreamConfigJava.SYSTEM, TEST_SYSTEM); - StreamUtil.getStreamSpec("", new StreamConfig1(config)); + StreamUtil.getStreamSpec("", new StreamConfigJava(config)); } // Null is not allowed for streamId. @Test(expected = IllegalArgumentException.class) public void testGetStreamStreamIdNull() { Config config = buildStreamConfig(null, - StreamConfig1.SYSTEM(), TEST_SYSTEM); + StreamConfigJava.SYSTEM, TEST_SYSTEM); - StreamUtil.getStreamSpec(null, new StreamConfig1(config)); + StreamUtil.getStreamSpec(null, new StreamConfigJava(config)); } @@ -311,7 +312,7 @@ public void testGetStreamStreamIdNull() { private Config buildStreamConfig(String streamId, String... kvs) { // inject streams.x. into each key for (int i = 0; i < kvs.length - 1; i += 2) { - kvs[i] = String.format(StreamConfig1.STREAM_ID_PREFIX(), streamId) + kvs[i]; + kvs[i] = String.format(StreamConfigJava.STREAM_ID_PREFIX, streamId) + kvs[i]; } return buildConfig(kvs); } diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index 1dceff875b..f6c66fe31a 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -23,18 +23,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; -import java.util.stream.Collectors; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.apache.kafka.clients.admin.AdminClient; @@ -57,7 +45,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.KafkaConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.config.SystemConfig; import org.apache.samza.startpoint.Startpoint; import org.apache.samza.startpoint.StartpointOldest; @@ -84,6 +72,19 @@ import scala.runtime.AbstractFunction2; import scala.runtime.BoxedUnit; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.stream.Collectors; + public class KafkaSystemAdmin implements SystemAdmin { private static final Logger LOG = LoggerFactory.getLogger(KafkaSystemAdmin.class); @@ -140,7 +141,9 @@ public KafkaSystemAdmin(String systemName, Config config, Consumer metadataConsu LOG.info("New admin client with props:" + props); adminClient = AdminClient.create(props); - KafkaConfig kafkaConfig = new KafkaConfig(config); + StreamConfigJava streamConfig = new StreamConfigJava(config); + + KafkaConfig kafkaConfig = new KafkaConfig(streamConfig); coordinatorStreamReplicationFactor = Integer.valueOf(kafkaConfig.getCoordinatorReplicationFactor()); coordinatorStreamProperties = getCoordinatorStreamProperties(kafkaConfig); @@ -687,7 +690,7 @@ static Map getIntermediateStreamProperties(Config config) { ApplicationConfig appConfig = new ApplicationConfig(config); if (appConfig.getAppMode() == ApplicationConfig.ApplicationMode.BATCH) { - StreamConfig1 streamConfig = new StreamConfig1(config); + StreamConfigJava streamConfig = new StreamConfigJava(config); intermedidateStreamProperties = JavaConverters.asJavaCollectionConverter(streamConfig.getStreamIds()) .asJavaCollection() .stream() diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala index 76df02e730..52d6354584 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala @@ -108,7 +108,7 @@ class KafkaSystemFactory extends SystemFactory with Logging { def getIntermediateStreamProperties(config: Config): Map[String, Properties] = { val appConfig = new ApplicationConfig(config) if (appConfig.getAppMode == ApplicationMode.BATCH) { - val streamConfig = new StreamConfig1(config) + val streamConfig = new StreamConfigJava(config) streamConfig.getStreamIds().filter(streamConfig.getIsIntermediateStream(_)).map(streamId => { // only the override here val properties = new Properties() diff --git a/samza-log4j/src/main/java/org/apache/samza/config/Log4jSystemConfig.java b/samza-log4j/src/main/java/org/apache/samza/config/Log4jSystemConfig.java index a7a334b50e..8d201367f6 100644 --- a/samza-log4j/src/main/java/org/apache/samza/config/Log4jSystemConfig.java +++ b/samza-log4j/src/main/java/org/apache/samza/config/Log4jSystemConfig.java @@ -21,6 +21,8 @@ import org.apache.samza.system.SystemStream; +import java.util.Optional; + /** * This class contains the methods for getting properties that are needed by the @@ -81,8 +83,8 @@ public String getSerdeClass(String name) { } public String getStreamSerdeName(String systemName, String streamName) { - StreamConfig1 streamConfig = new StreamConfig1(this); - scala.Option option = streamConfig.getStreamMsgSerde(new SystemStream(systemName, streamName)); - return option.isEmpty() ? null : option.get(); + StreamConfigJava streamConfig = new StreamConfigJava(this); + Optional option = streamConfig.getStreamMsgSerde(new SystemStream(systemName, streamName)); + return option.isPresent() ? option.get() : null; } } diff --git a/samza-log4j2/src/main/java/org/apache/samza/config/Log4jSystemConfig.java b/samza-log4j2/src/main/java/org/apache/samza/config/Log4jSystemConfig.java index a7a334b50e..8d201367f6 100644 --- a/samza-log4j2/src/main/java/org/apache/samza/config/Log4jSystemConfig.java +++ b/samza-log4j2/src/main/java/org/apache/samza/config/Log4jSystemConfig.java @@ -21,6 +21,8 @@ import org.apache.samza.system.SystemStream; +import java.util.Optional; + /** * This class contains the methods for getting properties that are needed by the @@ -81,8 +83,8 @@ public String getSerdeClass(String name) { } public String getStreamSerdeName(String systemName, String streamName) { - StreamConfig1 streamConfig = new StreamConfig1(this); - scala.Option option = streamConfig.getStreamMsgSerde(new SystemStream(systemName, streamName)); - return option.isEmpty() ? null : option.get(); + StreamConfigJava streamConfig = new StreamConfigJava(this); + Optional option = streamConfig.getStreamMsgSerde(new SystemStream(systemName, streamName)); + return option.isPresent() ? option.get() : null; } } diff --git a/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java b/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java index 51775fbfb5..991281adfd 100644 --- a/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java +++ b/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java @@ -28,7 +28,7 @@ import org.apache.commons.lang.Validate; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.table.descriptors.CachingTableDescriptor; import org.apache.samza.table.descriptors.RemoteTableDescriptor; import org.apache.samza.table.descriptors.TableDescriptor; @@ -100,11 +100,11 @@ public SqlIOConfig(String systemName, String streamName, List sourcePart if (!isRemoteTable()) { // The below config is required for local table and streams but not for remote table. - streamConfigs.put(String.format(StreamConfig1.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), streamName); + streamConfigs.put(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), streamName); if (tableDescriptor != null) { // For local table, set the bootstrap config and default offset to oldest - streamConfigs.put(String.format(StreamConfig1.BOOTSTRAP_FOR_STREAM_ID(), streamId), "true"); - streamConfigs.put(String.format(StreamConfig1.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), streamId), "oldest"); + streamConfigs.put(String.format(StreamConfigJava.BOOTSTRAP_FOR_STREAM_ID(), streamId), "true"); + streamConfigs.put(String.format(StreamConfigJava.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), streamId), "oldest"); } } diff --git a/samza-sql/src/test/java/org/apache/samza/sql/translator/TestQueryTranslator.java b/samza-sql/src/test/java/org/apache/samza/sql/translator/TestQueryTranslator.java index 4819d8b588..89ebad0d48 100644 --- a/samza-sql/src/test/java/org/apache/samza/sql/translator/TestQueryTranslator.java +++ b/samza-sql/src/test/java/org/apache/samza/sql/translator/TestQueryTranslator.java @@ -29,7 +29,7 @@ import org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.context.ContainerContext; import org.apache.samza.context.Context; import org.apache.samza.operators.OperatorSpecGraph; @@ -83,7 +83,7 @@ public void testTranslate() { translator.translate(queryInfo.get(0), appDesc, 0); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); - StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); + StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); @@ -122,7 +122,7 @@ public void testTranslateFanIn() { translator.translate(queryInfo.get(1), appDesc, 1); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); - StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); + StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); String inputStreamId1 = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem1 = streamConfig.getSystem(inputStreamId1); String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1); @@ -167,7 +167,7 @@ public void testTranslateFanOut() { translator.translate(queryInfo.get(1), appDesc, 1); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); - StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); + StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); @@ -211,7 +211,7 @@ public void testTranslateMultiSql() { translator.translate(queryInfo.get(1), appDesc, 1); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); - StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); + StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); String inputStreamId1 = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem1 = streamConfig.getSystem(inputStreamId1); String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1); @@ -259,7 +259,7 @@ public void testTranslateComplex() { translator.translate(queryInfo.get(0), streamAppDesc, 0); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); + StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); @@ -296,7 +296,7 @@ public void testTranslateSubQuery() { translator.translate(queryInfo.get(0), streamAppDesc, 0); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); + StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); @@ -631,7 +631,7 @@ public void testTranslateStreamTableInnerJoin() { translator.translate(queryInfo.get(0), streamAppDesc, 0); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); + StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); String input1StreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String input1System = streamConfig.getSystem(input1StreamId); String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId); @@ -699,7 +699,7 @@ public void testTranslateStreamTableLeftJoin() { OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); + StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); String input1StreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String input1System = streamConfig.getSystem(input1StreamId); String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId); @@ -766,7 +766,7 @@ public void testTranslateStreamTableRightJoin() { OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfig1 streamConfig = new StreamConfig1(samzaConfig); + StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); String input1StreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String input1System = streamConfig.getSystem(input1StreamId); String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId); diff --git a/samza-test/src/main/java/org/apache/samza/test/framework/TestRunner.java b/samza-test/src/main/java/org/apache/samza/test/framework/TestRunner.java index 9121662000..37b4300aaf 100644 --- a/samza-test/src/main/java/org/apache/samza/test/framework/TestRunner.java +++ b/samza-test/src/main/java/org/apache/samza/test/framework/TestRunner.java @@ -40,7 +40,7 @@ import org.apache.samza.config.JobConfig; import org.apache.samza.config.JobCoordinatorConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.config.TaskConfig; import org.apache.samza.container.grouper.task.SingleContainerGrouperFactory; import org.apache.samza.context.ExternalContext; @@ -431,9 +431,9 @@ private void deleteDirectory(String path) { * over {@link org.apache.samza.application.descriptors.ApplicationDescriptor} generated configs */ private void addSerdeConfigs(StreamDescriptor descriptor) { - String streamIdPrefix = String.format(StreamConfig1.STREAM_ID_PREFIX(), descriptor.getStreamId()); - String keySerdeConfigKey = streamIdPrefix + StreamConfig1.KEY_SERDE(); - String msgSerdeConfigKey = streamIdPrefix + StreamConfig1.MSG_SERDE(); + String streamIdPrefix = String.format(StreamConfigJava.STREAM_ID_PREFIX, descriptor.getStreamId()); + String keySerdeConfigKey = streamIdPrefix + StreamConfigJava.KEY_SERDE; + String msgSerdeConfigKey = streamIdPrefix + StreamConfigJava.MSG_SERDE; this.configs.put(keySerdeConfigKey, null); this.configs.put(msgSerdeConfigKey, null); } diff --git a/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java b/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java index 7368ea9f59..1aa3b7b474 100644 --- a/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java +++ b/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java @@ -36,7 +36,7 @@ import org.apache.samza.application.descriptors.StreamApplicationDescriptor; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.config.TaskConfig; import org.apache.samza.operators.OutputStream; import org.apache.samza.serializers.NoOpSerde; @@ -105,7 +105,7 @@ public void testDownstreamOperatorExceptionIsBubbledUp() { } private List runTest(List pageViews, Map configs) { - configs.put(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), PAGE_VIEW_STREAM), TEST_SYSTEM); + configs.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID(), PAGE_VIEW_STREAM), TEST_SYSTEM); InMemorySystemDescriptor isd = new InMemorySystemDescriptor(TEST_SYSTEM); InMemoryInputDescriptor pageViewStreamDesc = isd diff --git a/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTableWithSideInputsEndToEnd.java b/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTableWithSideInputsEndToEnd.java index 4f9dacd624..b63b81bcfb 100644 --- a/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTableWithSideInputsEndToEnd.java +++ b/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTableWithSideInputsEndToEnd.java @@ -29,7 +29,7 @@ import org.apache.samza.SamzaException; import org.apache.samza.application.StreamApplication; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.application.descriptors.StreamApplicationDescriptor; import org.apache.samza.operators.KV; import org.apache.samza.table.descriptors.TableDescriptor; @@ -82,9 +82,9 @@ public void testJoinWithDurableSideInputTable() { private void runTest(String systemName, StreamApplication app, List pageViews, List profiles) { Map configs = new HashMap<>(); - configs.put(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), PAGEVIEW_STREAM), systemName); - configs.put(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), PROFILE_STREAM), systemName); - configs.put(String.format(StreamConfig1.SYSTEM_FOR_STREAM_ID(), ENRICHED_PAGEVIEW_STREAM), systemName); + configs.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, PAGEVIEW_STREAM), systemName); + configs.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, PROFILE_STREAM), systemName); + configs.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, ENRICHED_PAGEVIEW_STREAM), systemName); InMemorySystemDescriptor isd = new InMemorySystemDescriptor(systemName); From c5791ac4ab95e07e21e2b6a6bca7d3e7eb2ea540 Mon Sep 17 00:00:00 2001 From: Boris S Date: Thu, 26 Sep 2019 13:33:36 -0700 Subject: [PATCH 3/9] added file --- .../apache/samza/config/StreamConfigJava.java | 347 ++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java diff --git a/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java b/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java new file mode 100644 index 0000000000..d7a995e626 --- /dev/null +++ b/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java @@ -0,0 +1,347 @@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.samza.config; + + import com.google.common.collect.Sets; + import org.apache.samza.system.SystemStream; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + + import java.lang.invoke.MethodHandles; + import java.util.Arrays; + import java.util.Collections; + import java.util.List; + import java.util.Map; + import java.util.Optional; + import java.util.Set; + import java.util.stream.Collectors; + + public class StreamConfigJava extends MapConfig { + public static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + + // Samza configs for streams + public static final String SAMZA_PROPERTY = "samza."; + public static final String SYSTEM = SAMZA_PROPERTY + "system"; + public static final String PHYSICAL_NAME = SAMZA_PROPERTY + "physical.name"; + public static final String MSG_SERDE = SAMZA_PROPERTY + "msg.serde"; + public static final String KEY_SERDE = SAMZA_PROPERTY + "key.serde"; + public static final String CONSUMER_RESET_OFFSET = SAMZA_PROPERTY + "reset.offset"; + public static final String CONSUMER_OFFSET_DEFAULT = SAMZA_PROPERTY + "offset.default"; + public static final String BOOTSTRAP = SAMZA_PROPERTY + "bootstrap"; + public static final String PRIORITY = SAMZA_PROPERTY + "priority"; + public static final String IS_INTERMEDIATE = SAMZA_PROPERTY + "intermediate"; + public static final String DELETE_COMMITTED_MESSAGES = SAMZA_PROPERTY + "delete.committed.messages"; + public static final String IS_BOUNDED = SAMZA_PROPERTY + "bounded"; + public static final String BROADCAST = SAMZA_PROPERTY + "broadcast"; + + // We don't want any external dependencies on these patterns while both exist. Use getProperty to ensure proper values. + private static final String STREAMS_PREFIX = "streams."; + + public static final String STREAM_PREFIX = "systems.%s.streams.%s."; + public static final String STREAM_ID_PREFIX = STREAMS_PREFIX + "%s."; + public static final String SYSTEM_FOR_STREAM_ID = STREAM_ID_PREFIX + SYSTEM; + public static final String PHYSICAL_NAME_FOR_STREAM_ID = STREAM_ID_PREFIX + PHYSICAL_NAME; + public static final String IS_INTERMEDIATE_FOR_STREAM_ID = STREAM_ID_PREFIX + IS_INTERMEDIATE; + public static final String DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID = STREAM_ID_PREFIX + DELETE_COMMITTED_MESSAGES; + public static final String IS_BOUNDED_FOR_STREAM_ID = STREAM_ID_PREFIX + IS_BOUNDED; + public static final String PRIORITY_FOR_STREAM_ID = STREAM_ID_PREFIX + PRIORITY; + public static final String CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID = STREAM_ID_PREFIX + CONSUMER_OFFSET_DEFAULT; + public static final String BOOTSTRAP_FOR_STREAM_ID = STREAM_ID_PREFIX + BOOTSTRAP; + public static final String BROADCAST_FOR_STREAM_ID = STREAM_ID_PREFIX + BROADCAST; + + //implicit def Config2Stream(config: Config) = new StreamConfig(config) + + private Optional nonEmptyOption(String value) { + if (value == null || value.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(value); + } + } + + public StreamConfigJava(Config config) { + super(config); + } + + + /** + * Gets the stream properties from the legacy config style: + * systems.{system}.streams.{streams}.* + * + * @param systemName the system name under which the properties are configured + * @param streamName the stream name + * @return the map of properties for the stream + */ + private Map getSystemStreamProperties(String systemName, String streamName) { + if (systemName == null) { + Collections.emptyMap(); + } + SystemConfig systemConfig = new SystemConfig(this); + Config defaults = systemConfig.getDefaultStreamProperties(systemName); + Config explicitConfigs = subset(String.format(STREAM_PREFIX, systemName, streamName), true); + return new MapConfig(defaults, explicitConfigs); + } + + + /** + * Gets the properties for the specified streamId from the config. + * It first applies any legacy configs from this config location: + * systems.{system}.streams.{stream}.* + *

+ * It then overrides them with properties of the new config format: + * streams.{streamId}.* + * + * @param streamId the identifier for the stream in the config. + * @return the merged map of config properties from both the legacy and new config styles + */ + private MapConfig getAllStreamProperties(String streamId) { + Config allProperties = subset(String.format(STREAM_ID_PREFIX, streamId)); + Map inheritedLegacyProperties = getSystemStreamProperties(getSystem(streamId), getPhysicalName(streamId)); + return new MapConfig(Arrays.asList(inheritedLegacyProperties, allProperties)); + } + + /** + * Gets the distinct stream IDs of all the streams defined in the config + * + * @return collection of stream IDs + */ + public List getStreamIds() { + // StreamIds are not allowed to have '.' so the first index of '.' marks the end of the streamId. + return subset(STREAMS_PREFIX).keySet().stream().map(key -> key.substring(0, key.indexOf("."))) + .distinct().collect(Collectors.toList()); + } + + private List getStreamIdsForSystem(String system) { + return getStreamIds().stream().filter(streamId -> system.equals(getSystem(streamId))).collect(Collectors.toList()); + } + + public String systemStreamToStreamId(SystemStream systemStream) { + List streamIds = getStreamIdsForSystem(systemStream.getSystem()).stream() + .filter(streamId -> systemStream.getStream().equals(getPhysicalName(streamId))).collect(Collectors.toList()); + if (streamIds.size() > 1) { + throw new IllegalStateException(String.format("There was more than one stream found for system stream %s", systemStream)); + } + + return streamIds.isEmpty() ? null : streamIds.get(0); + } + + /** + * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza + * interacts with the stream, as opposed to a property of the stream itself. + *

+ * Note, because the translation is not perfect between SystemStream and streamId, + * this method is not identical to getProperty(streamId, property) + * + * @param systemStream the SystemStream for which the property value will be retrieved. + * @param property the samza property name excluding the leading delimiter. e.g. "samza.x.y" + */ + protected String getSamzaProperty(SystemStream systemStream, String property) { + if (!property.startsWith(SAMZA_PROPERTY)) { + throw new IllegalArgumentException( + String.format("Attempt to fetch a non samza property for SystemStream %s named %s", systemStream, property)); + } + + String streamVal = getAllStreamProperties(systemStreamToStreamId(systemStream)).get(property); + return (streamVal != null) ? streamVal : getSystemStreamProperties(systemStream.getSystem(), systemStream.getStream()).get(property); + } + + /** + * Gets the System associated with the specified streamId. + * It first looks for the property + * streams.{streamId}.system + *

+ * If no value was provided, it uses + * job.default.system + * + * @param streamId the identifier for the stream in the config. + * @return the system name associated with the stream. + */ + public String getSystem(String streamId) { + return getOrDefault(String.format(SYSTEM_FOR_STREAM_ID, streamId), + new JobConfig(this).getDefaultSystem().orElse(null)); + } + + /** + * Gets the physical name for the specified streamId. + * + * @param streamId the identifier for the stream in the config. + * @return the physical identifier for the stream or the default if it is undefined. + */ + public String getPhysicalName(String streamId) { + // use streamId as the default physical name + return get(String.format(PHYSICAL_NAME_FOR_STREAM_ID, streamId, streamId)); + } + + + public Optional getStreamMsgSerde(SystemStream systemStream) { + return nonEmptyOption(getSamzaProperty(systemStream, MSG_SERDE)); + } + + public Optional getStreamKeySerde(SystemStream systemStream) { + return nonEmptyOption(getSamzaProperty(systemStream, KEY_SERDE)); + } + + public boolean getResetOffset(SystemStream systemStream) { + String resetOffset = getSamzaProperty(systemStream, CONSUMER_RESET_OFFSET); + if (!resetOffset.equalsIgnoreCase("true") && !resetOffset.equalsIgnoreCase("false")) { + LOG.warn("Got a .samza.reset.offset configuration for SystemStream {} that is not true or false (was {})." + + " Defaulting to false.", systemStream, resetOffset); + + resetOffset = "false"; + } + return Boolean.valueOf(resetOffset); + } + + + /** + * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza + * interacts with the stream, as opposed to a property of the stream itself. + *

+ * Note, because the translation is not perfect between SystemStream and streamId, + * this method is not identical to getProperty(streamId, property) + * + * @param systemStream the SystemStream for which the property value will be retrieved. + * @param property the samza property name excluding the leading delimiter. e.g. "samza.x.y" + * @param defaultValue the default value to use if the property value is not found + */ + protected String getSamzaProperty(SystemStream systemStream, String property, String defaultValue) { + String streamVal = getSamzaProperty(systemStream, property); + + return streamVal != null ? streamVal : defaultValue; + } + + /** + * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza + * interacts with the stream, as opposed to a property of the stream itself. + *

+ * Note, because the translation is not perfect between SystemStream and streamId, + * this method is not identical to getProperty(streamId, property) + */ + protected boolean containsSamzaProperty(SystemStream systemStream, String property) { + if (!property.startsWith(SAMZA_PROPERTY)) { + throw new IllegalArgumentException( + String.format("Attempt to fetch a non samza property for SystemStream %s named %s", systemStream, property)); + } + return getSamzaProperty(systemStream, property) != null; + } + + public boolean isResetOffsetConfigured(SystemStream systemStream) { + return containsSamzaProperty(systemStream, CONSUMER_RESET_OFFSET); + } + + public String getDefaultStreamOffset(SystemStream systemStream) { + return getSamzaProperty(systemStream, CONSUMER_OFFSET_DEFAULT); + } + + public boolean isDefaultStreamOffsetConfigured(SystemStream systemStream) { + return containsSamzaProperty(systemStream, CONSUMER_OFFSET_DEFAULT); + } + + public boolean getBootstrapEnabled(SystemStream systemStream) { + return Boolean.parseBoolean(getSamzaProperty(systemStream, BOOTSTRAP)); + } + + public boolean getBroadcastEnabled(SystemStream systemStream) { + return java.lang.Boolean.parseBoolean(getSamzaProperty(systemStream, BROADCAST)); + } + + public int getPriority(SystemStream systemStream) { + return Integer.parseInt(getSamzaProperty(systemStream, PRIORITY, "-1")); + } + + /** + * A streamId is translated to a SystemStream by looking up its System and physicalName. It + * will use the streamId as the stream name if the physicalName doesn't exist. + */ + public SystemStream streamIdToSystemStream(String streamId) { + return new SystemStream(getSystem(streamId), getPhysicalName(streamId)); + } + + + /** + * Returns a list of all SystemStreams that have a serde defined from the config file. + */ + public Set getSerdeStreams(String systemName) { + Config subConf = subset(String.format("systems.%s.streams.", systemName, true)); + Set legacySystemStreams = subConf.keySet().stream() + .filter(k -> k.endsWith(MSG_SERDE) || k.endsWith(KEY_SERDE)) + .map(k -> { + String streamName = k.substring(0, k.length() - 16 /* .samza.XXX.serde length */); + return new SystemStream(systemName, streamName); + }) + .collect(Collectors.toSet()); + + Set systemStreams = subset(STREAMS_PREFIX).keySet().stream() + .filter(k -> k.endsWith(MSG_SERDE) || k.endsWith(KEY_SERDE)) + .map(k -> k.substring(0, k.length() - 16 /* .samza.XXX.serde length */)) + .filter(streamId -> systemName.equals(getSystem(streamId))) + .map(streamId -> streamIdToSystemStream(streamId)).collect(Collectors.toSet()); + + return Sets.union(legacySystemStreams, systemStreams).immutableCopy(); + } + + /** + * Gets the properties for the specified streamId from the config. + * It first applies any legacy configs from this config location: + * systems.{system}.streams.{stream}.* + *

+ * It then overrides them with properties of the new config format: + * streams.{streamId}.* + *

+ * Only returns properties of the stream itself, not any of the samza properties for the stream. + * + * @param streamId the identifier for the stream in the config. + * @return the merged map of config properties from both the legacy and new config styles + */ + public Config getStreamProperties(String streamId) { + MapConfig allProperties = getAllStreamProperties(streamId); + Config samzaProperties = allProperties.subset(SAMZA_PROPERTY, false); + Map filteredStreamProperties = + allProperties.entrySet().stream().filter(kv -> !samzaProperties.containsKey(kv.getKey())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return new MapConfig(filteredStreamProperties); + } + + + /** + * Gets the boolean flag of whether the specified streamId is an intermediate stream + * + * @param streamId the identifier for the stream in the config. + * @return true if the stream is intermediate + */ + public boolean getIsIntermediateStream(String streamId) { + return getBoolean(String.format(IS_INTERMEDIATE_FOR_STREAM_ID, streamId), false); + } + + /** + * Gets the boolean flag of whether the committed messages specified streamId can be deleted + * + * @param streamId the identifier for the stream in the config. + * @return true if the committed messages of the stream can be deleted + */ + public boolean getDeleteCommittedMessages(String streamId) { + return getBoolean(String.format(DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, streamId), false); + } + + public boolean getIsBounded(String streamId) { + return getBoolean(String.format(IS_BOUNDED_FOR_STREAM_ID, streamId), false); + } + + } From f8623589786cd0ff25fc3d44025cf33eb775a48d Mon Sep 17 00:00:00 2001 From: Boris S Date: Thu, 26 Sep 2019 13:37:10 -0700 Subject: [PATCH 4/9] more cleanup --- .../execution/JobNodeConfigurationGenerator.java | 10 +++++----- .../org/apache/samza/execution/StreamEdge.java | 14 +++++++------- .../org/apache/samza/config/TestSystemConfig.java | 2 +- .../samza/system/kafka/KafkaSystemAdmin.java | 6 ++---- .../apache/samza/sql/interfaces/SqlIOConfig.java | 6 +++--- .../samza/test/operator/TestAsyncFlatMap.java | 2 +- 6 files changed, 19 insertions(+), 21 deletions(-) diff --git a/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java b/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java index 4d78ea7d28..47eca30c8e 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java +++ b/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java @@ -251,7 +251,7 @@ private void configureTables(Map generatedConfig, Config origina .map(sideInput -> StreamUtil.getSystemStreamFromNameOrId(originalConfig, sideInput)) .forEach(systemStream -> { inputs.add(StreamUtil.getNameFromSystemStream(systemStream)); - generatedConfig.put(String.format(StreamConfigJava.STREAM_PREFIX() + StreamConfigJava.BOOTSTRAP(), + generatedConfig.put(String.format(StreamConfigJava.STREAM_PREFIX + StreamConfigJava.BOOTSTRAP, systemStream.getSystem(), systemStream.getStream()), "true"); }); } @@ -314,14 +314,14 @@ private void configureSerdes(Map configs, Map { - String streamIdPrefix = String.format(StreamConfigJava.STREAM_ID_PREFIX(), streamId); - String keySerdeConfigKey = streamIdPrefix + StreamConfigJava.KEY_SERDE(); + String streamIdPrefix = String.format(StreamConfigJava.STREAM_ID_PREFIX, streamId); + String keySerdeConfigKey = streamIdPrefix + StreamConfigJava.KEY_SERDE; configs.put(keySerdeConfigKey, serdeUUIDs.get(serde)); }); streamMsgSerdes.forEach((streamId, serde) -> { - String streamIdPrefix = String.format(StreamConfigJava.STREAM_ID_PREFIX(), streamId); - String valueSerdeConfigKey = streamIdPrefix + StreamConfigJava.MSG_SERDE(); + String streamIdPrefix = String.format(StreamConfigJava.STREAM_ID_PREFIX, streamId); + String valueSerdeConfigKey = streamIdPrefix + StreamConfigJava.MSG_SERDE; configs.put(valueSerdeConfigKey, serdeUUIDs.get(serde)); }); diff --git a/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java b/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java index c3d87bd769..915508f377 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java +++ b/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java @@ -122,21 +122,21 @@ Config generateConfig() { Map streamConfig = new HashMap<>(); StreamSpec spec = getStreamSpec(); String streamId = spec.getId(); - streamConfig.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID(), streamId), spec.getSystemName()); - streamConfig.put(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), spec.getPhysicalName()); + streamConfig.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, streamId), spec.getSystemName()); + streamConfig.put(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, streamId), spec.getPhysicalName()); if (isIntermediate()) { - streamConfig.put(String.format(StreamConfigJava.IS_INTERMEDIATE_FOR_STREAM_ID(), streamId), "true"); - streamConfig.put(String.format(StreamConfigJava.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), streamId), "true"); + streamConfig.put(String.format(StreamConfigJava.IS_INTERMEDIATE_FOR_STREAM_ID, streamId), "true"); + streamConfig.put(String.format(StreamConfigJava.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, streamId), "true"); // Setting offset.default to oldest only if the job is running in batch mode if (ApplicationConfig.ApplicationMode.BATCH.equals(new ApplicationConfig(config).getAppMode())) { - streamConfig.put(String.format(StreamConfigJava.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), streamId), "oldest"); + streamConfig.put(String.format(StreamConfigJava.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID, streamId), "oldest"); } - streamConfig.put(String.format(StreamConfigJava.PRIORITY_FOR_STREAM_ID(), streamId), String.valueOf(Integer.MAX_VALUE)); + streamConfig.put(String.format(StreamConfigJava.PRIORITY_FOR_STREAM_ID, streamId), String.valueOf(Integer.MAX_VALUE)); } spec.getConfig().forEach((property, value) -> { - streamConfig.put(String.format(StreamConfigJava.STREAM_ID_PREFIX(), streamId) + property, value); + streamConfig.put(String.format(StreamConfigJava.STREAM_ID_PREFIX, streamId) + property, value); }); return new MapConfig(streamConfig); diff --git a/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java b/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java index 12a65a50b8..36ecf58f5c 100644 --- a/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java +++ b/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java @@ -130,7 +130,7 @@ public void testGetDefaultStreamProperties() { new MapConfig(ImmutableMap.of(system1ConfigKey, system1ConfigValue, system2ConfigKey, system2ConfigValue)); SystemConfig systemConfig = new SystemConfig(config); assertEquals(expected, systemConfig.getDefaultStreamProperties(MOCK_SYSTEM_NAME1)); - assertEquals(new MapConfig, systemConfig.getDefaultStreamProperties(MOCK_SYSTEM_NAME2)); + assertEquals(new MapConfig(), systemConfig.getDefaultStreamProperties(MOCK_SYSTEM_NAME2)); } @Test diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index f6c66fe31a..78c26f360a 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -147,8 +147,7 @@ public KafkaSystemAdmin(String systemName, Config config, Consumer metadataConsu coordinatorStreamReplicationFactor = Integer.valueOf(kafkaConfig.getCoordinatorReplicationFactor()); coordinatorStreamProperties = getCoordinatorStreamProperties(kafkaConfig); - Map storeToChangelog = - JavaConverters.mapAsJavaMapConverter(kafkaConfig.getKafkaChangelogEnabledStores()).asJava(); + Map storeToChangelog = kafkaConfig.getKafkaChangelogEnabledStores(); // Construct the meta information for each topic, if the replication factor is not defined, // we use 2 (DEFAULT_REPL_FACTOR) as the number of replicas for the change log stream. changelogTopicMetaInformation = new HashMap<>(); @@ -691,8 +690,7 @@ static Map getIntermediateStreamProperties(Config config) { if (appConfig.getAppMode() == ApplicationConfig.ApplicationMode.BATCH) { StreamConfigJava streamConfig = new StreamConfigJava(config); - intermedidateStreamProperties = JavaConverters.asJavaCollectionConverter(streamConfig.getStreamIds()) - .asJavaCollection() + intermedidateStreamProperties = streamConfig.getStreamIds() .stream() .filter(streamConfig::getIsIntermediateStream) .collect(Collectors.toMap(Function.identity(), streamId -> { diff --git a/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java b/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java index 991281adfd..c0a8a4e8e2 100644 --- a/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java +++ b/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java @@ -100,11 +100,11 @@ public SqlIOConfig(String systemName, String streamName, List sourcePart if (!isRemoteTable()) { // The below config is required for local table and streams but not for remote table. - streamConfigs.put(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), streamName); + streamConfigs.put(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, streamId), streamName); if (tableDescriptor != null) { // For local table, set the bootstrap config and default offset to oldest - streamConfigs.put(String.format(StreamConfigJava.BOOTSTRAP_FOR_STREAM_ID(), streamId), "true"); - streamConfigs.put(String.format(StreamConfigJava.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), streamId), "oldest"); + streamConfigs.put(String.format(StreamConfigJava.BOOTSTRAP_FOR_STREAM_ID, streamId), "true"); + streamConfigs.put(String.format(StreamConfigJava.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID, streamId), "oldest"); } } diff --git a/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java b/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java index 1aa3b7b474..03f3894a56 100644 --- a/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java +++ b/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java @@ -105,7 +105,7 @@ public void testDownstreamOperatorExceptionIsBubbledUp() { } private List runTest(List pageViews, Map configs) { - configs.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID(), PAGE_VIEW_STREAM), TEST_SYSTEM); + configs.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, PAGE_VIEW_STREAM), TEST_SYSTEM); InMemorySystemDescriptor isd = new InMemorySystemDescriptor(TEST_SYSTEM); InMemoryInputDescriptor pageViewStreamDesc = isd From 135b17c71b0dfa29458ab437f629e1df8d6e842f Mon Sep 17 00:00:00 2001 From: Boris S Date: Thu, 26 Sep 2019 16:24:34 -0700 Subject: [PATCH 5/9] updates --- .../samza/system/eventhub/EventHubConfig.java | 1 - .../apache/samza/execution/StreamManager.java | 1 - .../org/apache/samza/util/StreamUtil.java | 1 - .../samza/checkpoint/OffsetManager.scala | 11 +- .../apache/samza/config/StreamConfigJava.java | 601 +++++++++--------- .../samza/container/SamzaContainer.scala | 18 +- .../apache/samza/container/TaskInstance.scala | 4 +- .../MetricsSnapshotReporterFactory.scala | 2 +- .../apache/samza/config/TestStreamConfig.java | 28 +- .../samza/execution/TestExecutionPlanner.java | 4 +- .../samza/testUtils/StreamTestUtils.java | 6 +- .../org/apache/samza/config/KafkaConfig.scala | 5 +- .../system/kafka/KafkaConsumerProxy.java | 4 + .../system/kafka/KafkaSystemFactory.scala | 3 +- .../apache/samza/config/TestKafkaConfig.scala | 12 +- .../samza/sql/SamzaSqlInputMessage.java | 6 +- .../samza/sql/planner/SamzaSqlValidator.java | 2 +- 17 files changed, 357 insertions(+), 352 deletions(-) diff --git a/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java b/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java index e755cb58aa..2bd52d1d95 100644 --- a/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java +++ b/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java @@ -28,7 +28,6 @@ import org.apache.samza.system.eventhub.producer.EventHubSystemProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.collection.JavaConversions; import java.time.Duration; import java.util.HashMap; diff --git a/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java b/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java index 5429847cfa..2dab3445b1 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java +++ b/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java @@ -40,7 +40,6 @@ import org.apache.samza.util.StreamUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.collection.JavaConversions; public class StreamManager { diff --git a/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java b/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java index 14b57bfb1a..f22b63a85a 100644 --- a/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java +++ b/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java @@ -21,7 +21,6 @@ import org.apache.samza.SamzaException; import org.apache.samza.config.Config; import org.apache.samza.config.StreamConfigJava; -import org.apache.samza.config.StreamConfigJava; import org.apache.samza.system.StreamSpec; import org.apache.samza.system.SystemStream; diff --git a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala index 4cb6ce7bce..8a23f892f8 100644 --- a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala +++ b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala @@ -25,8 +25,7 @@ import java.util.concurrent.ConcurrentHashMap import org.apache.commons.lang3.StringUtils import org.apache.samza.SamzaException import org.apache.samza.annotation.InterfaceStability -import org.apache.samza.config.StreamConfig1.Config2Stream -import org.apache.samza.config.{Config, SystemConfig} +import org.apache.samza.config.{Config, StreamConfigJava, SystemConfig} import org.apache.samza.container.TaskName import org.apache.samza.startpoint.{Startpoint, StartpointManager} import org.apache.samza.system.SystemStreamMetadata.OffsetType @@ -88,8 +87,10 @@ object OffsetManager extends Logging { // Get default offset. val streamDefaultOffset = streamConfig.getDefaultStreamOffset(systemStream) val systemDefaultOffset = new SystemConfig(streamConfig).getSystemOffsetDefault(systemStream.getSystem) - val defaultOffsetType = if (streamDefaultOffset != null) { - OffsetType.valueOf(streamDefaultOffset.toUpperCase) + val defaultOffsetType = if (streamDefaultOffset.isPresent) { + OffsetType.valueOf(streamDefaultOffset.get.toUpperCase) + } else if (systemDefaultOffset != null) { + OffsetType.valueOf(systemDefaultOffset.toUpperCase) } else { info("No default offset for %s defined. Using upcoming." format systemStream) OffsetType.UPCOMING @@ -97,7 +98,7 @@ object OffsetManager extends Logging { debug("Using default offset %s for %s." format (defaultOffsetType, systemStream)) // Get reset offset. - val resetOffset = config.getResetOffset(systemStream) + val resetOffset = streamConfig.getResetOffset(systemStream) debug("Using reset offset %s for %s." format (resetOffset, systemStream)) // Build OffsetSetting so we can create a map for OffsetManager. diff --git a/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java b/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java index d7a995e626..889bc4fbc6 100644 --- a/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java +++ b/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java @@ -1,347 +1,346 @@ - /* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - package org.apache.samza.config; - - import com.google.common.collect.Sets; - import org.apache.samza.system.SystemStream; - import org.slf4j.Logger; - import org.slf4j.LoggerFactory; - - import java.lang.invoke.MethodHandles; - import java.util.Arrays; - import java.util.Collections; - import java.util.List; - import java.util.Map; - import java.util.Optional; - import java.util.Set; - import java.util.stream.Collectors; - - public class StreamConfigJava extends MapConfig { - public static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - - // Samza configs for streams - public static final String SAMZA_PROPERTY = "samza."; - public static final String SYSTEM = SAMZA_PROPERTY + "system"; - public static final String PHYSICAL_NAME = SAMZA_PROPERTY + "physical.name"; - public static final String MSG_SERDE = SAMZA_PROPERTY + "msg.serde"; - public static final String KEY_SERDE = SAMZA_PROPERTY + "key.serde"; - public static final String CONSUMER_RESET_OFFSET = SAMZA_PROPERTY + "reset.offset"; - public static final String CONSUMER_OFFSET_DEFAULT = SAMZA_PROPERTY + "offset.default"; - public static final String BOOTSTRAP = SAMZA_PROPERTY + "bootstrap"; - public static final String PRIORITY = SAMZA_PROPERTY + "priority"; - public static final String IS_INTERMEDIATE = SAMZA_PROPERTY + "intermediate"; - public static final String DELETE_COMMITTED_MESSAGES = SAMZA_PROPERTY + "delete.committed.messages"; - public static final String IS_BOUNDED = SAMZA_PROPERTY + "bounded"; - public static final String BROADCAST = SAMZA_PROPERTY + "broadcast"; - - // We don't want any external dependencies on these patterns while both exist. Use getProperty to ensure proper values. - private static final String STREAMS_PREFIX = "streams."; - - public static final String STREAM_PREFIX = "systems.%s.streams.%s."; - public static final String STREAM_ID_PREFIX = STREAMS_PREFIX + "%s."; - public static final String SYSTEM_FOR_STREAM_ID = STREAM_ID_PREFIX + SYSTEM; - public static final String PHYSICAL_NAME_FOR_STREAM_ID = STREAM_ID_PREFIX + PHYSICAL_NAME; - public static final String IS_INTERMEDIATE_FOR_STREAM_ID = STREAM_ID_PREFIX + IS_INTERMEDIATE; - public static final String DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID = STREAM_ID_PREFIX + DELETE_COMMITTED_MESSAGES; - public static final String IS_BOUNDED_FOR_STREAM_ID = STREAM_ID_PREFIX + IS_BOUNDED; - public static final String PRIORITY_FOR_STREAM_ID = STREAM_ID_PREFIX + PRIORITY; - public static final String CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID = STREAM_ID_PREFIX + CONSUMER_OFFSET_DEFAULT; - public static final String BOOTSTRAP_FOR_STREAM_ID = STREAM_ID_PREFIX + BOOTSTRAP; - public static final String BROADCAST_FOR_STREAM_ID = STREAM_ID_PREFIX + BROADCAST; - - //implicit def Config2Stream(config: Config) = new StreamConfig(config) - - private Optional nonEmptyOption(String value) { - if (value == null || value.isEmpty()) { - return Optional.empty(); - } else { - return Optional.of(value); - } +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.config; + +import com.google.common.collect.Sets; +import org.apache.samza.system.SystemStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.invoke.MethodHandles; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class StreamConfigJava extends MapConfig { + public static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + + // Samza configs for streams + public static final String SAMZA_PROPERTY = "samza."; + public static final String SYSTEM = SAMZA_PROPERTY + "system"; + public static final String PHYSICAL_NAME = SAMZA_PROPERTY + "physical.name"; + public static final String MSG_SERDE = SAMZA_PROPERTY + "msg.serde"; + public static final String KEY_SERDE = SAMZA_PROPERTY + "key.serde"; + public static final String CONSUMER_RESET_OFFSET = SAMZA_PROPERTY + "reset.offset"; + public static final String CONSUMER_OFFSET_DEFAULT = SAMZA_PROPERTY + "offset.default"; + public static final String BOOTSTRAP = SAMZA_PROPERTY + "bootstrap"; + public static final String PRIORITY = SAMZA_PROPERTY + "priority"; + public static final String IS_INTERMEDIATE = SAMZA_PROPERTY + "intermediate"; + public static final String DELETE_COMMITTED_MESSAGES = SAMZA_PROPERTY + "delete.committed.messages"; + public static final String IS_BOUNDED = SAMZA_PROPERTY + "bounded"; + public static final String BROADCAST = SAMZA_PROPERTY + "broadcast"; + + // We don't want any external dependencies on these patterns while both exist. Use getProperty to ensure proper values. + private static final String STREAMS_PREFIX = "streams."; + + public static final String STREAM_PREFIX = "systems.%s.streams.%s."; + public static final String STREAM_ID_PREFIX = STREAMS_PREFIX + "%s."; + public static final String SYSTEM_FOR_STREAM_ID = STREAM_ID_PREFIX + SYSTEM; + public static final String PHYSICAL_NAME_FOR_STREAM_ID = STREAM_ID_PREFIX + PHYSICAL_NAME; + public static final String IS_INTERMEDIATE_FOR_STREAM_ID = STREAM_ID_PREFIX + IS_INTERMEDIATE; + public static final String DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID = STREAM_ID_PREFIX + DELETE_COMMITTED_MESSAGES; + public static final String IS_BOUNDED_FOR_STREAM_ID = STREAM_ID_PREFIX + IS_BOUNDED; + public static final String PRIORITY_FOR_STREAM_ID = STREAM_ID_PREFIX + PRIORITY; + public static final String CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID = STREAM_ID_PREFIX + CONSUMER_OFFSET_DEFAULT; + public static final String BOOTSTRAP_FOR_STREAM_ID = STREAM_ID_PREFIX + BOOTSTRAP; + public static final String BROADCAST_FOR_STREAM_ID = STREAM_ID_PREFIX + BROADCAST; + + //implicit def Config2Stream(config: Config) = new StreamConfig(config) + + private Optional nonEmptyOption(String value) { + if (value == null || value.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(value); } + } - public StreamConfigJava(Config config) { - super(config); - } + public StreamConfigJava(Config config) { + super(config); + } - /** - * Gets the stream properties from the legacy config style: - * systems.{system}.streams.{streams}.* - * - * @param systemName the system name under which the properties are configured - * @param streamName the stream name - * @return the map of properties for the stream - */ - private Map getSystemStreamProperties(String systemName, String streamName) { - if (systemName == null) { - Collections.emptyMap(); - } - SystemConfig systemConfig = new SystemConfig(this); - Config defaults = systemConfig.getDefaultStreamProperties(systemName); - Config explicitConfigs = subset(String.format(STREAM_PREFIX, systemName, streamName), true); - return new MapConfig(defaults, explicitConfigs); + /** + * Gets the stream properties from the legacy config style: + * systems.{system}.streams.{streams}.* + * + * @param systemName the system name under which the properties are configured + * @param streamName the stream name + * @return the map of properties for the stream + */ + private Map getSystemStreamProperties(String systemName, String streamName) { + if (systemName == null) { + Collections.emptyMap(); } + SystemConfig systemConfig = new SystemConfig(this); + Config defaults = systemConfig.getDefaultStreamProperties(systemName); + Config explicitConfigs = subset(String.format(STREAM_PREFIX, systemName, streamName), true); + return new MapConfig(defaults, explicitConfigs); + } - /** - * Gets the properties for the specified streamId from the config. - * It first applies any legacy configs from this config location: - * systems.{system}.streams.{stream}.* - *

- * It then overrides them with properties of the new config format: - * streams.{streamId}.* - * - * @param streamId the identifier for the stream in the config. - * @return the merged map of config properties from both the legacy and new config styles - */ - private MapConfig getAllStreamProperties(String streamId) { - Config allProperties = subset(String.format(STREAM_ID_PREFIX, streamId)); - Map inheritedLegacyProperties = getSystemStreamProperties(getSystem(streamId), getPhysicalName(streamId)); - return new MapConfig(Arrays.asList(inheritedLegacyProperties, allProperties)); - } + /** + * Gets the properties for the specified streamId from the config. + * It first applies any legacy configs from this config location: + * systems.{system}.streams.{stream}.* + *

+ * It then overrides them with properties of the new config format: + * streams.{streamId}.* + * + * @param streamId the identifier for the stream in the config. + * @return the merged map of config properties from both the legacy and new config styles + */ + private MapConfig getAllStreamProperties(String streamId) { + Config allProperties = subset(String.format(STREAM_ID_PREFIX, streamId)); + Map inheritedLegacyProperties = getSystemStreamProperties(getSystem(streamId), getPhysicalName(streamId)); + return new MapConfig(Arrays.asList(inheritedLegacyProperties, allProperties)); + } - /** - * Gets the distinct stream IDs of all the streams defined in the config - * - * @return collection of stream IDs - */ - public List getStreamIds() { - // StreamIds are not allowed to have '.' so the first index of '.' marks the end of the streamId. - return subset(STREAMS_PREFIX).keySet().stream().map(key -> key.substring(0, key.indexOf("."))) - .distinct().collect(Collectors.toList()); - } + /** + * Gets the distinct stream IDs of all the streams defined in the config + * + * @return collection of stream IDs + */ + public Set getStreamIds() { + // StreamIds are not allowed to have '.' so the first index of '.' marks the end of the streamId. + return subset(STREAMS_PREFIX).keySet().stream().map(key -> key.substring(0, key.indexOf("."))) + .distinct().collect(Collectors.toSet()); + } + + private List getStreamIdsForSystem(String system) { + return getStreamIds().stream().filter(streamId -> system.equals(getSystem(streamId))).collect(Collectors.toList()); + } - private List getStreamIdsForSystem(String system) { - return getStreamIds().stream().filter(streamId -> system.equals(getSystem(streamId))).collect(Collectors.toList()); + public String systemStreamToStreamId(SystemStream systemStream) { + List streamIds = getStreamIdsForSystem(systemStream.getSystem()).stream() + .filter(streamId -> systemStream.getStream().equals(getPhysicalName(streamId))).collect(Collectors.toList()); + if (streamIds.size() > 1) { + throw new IllegalStateException(String.format("There was more than one stream found for system stream %s", systemStream)); } - public String systemStreamToStreamId(SystemStream systemStream) { - List streamIds = getStreamIdsForSystem(systemStream.getSystem()).stream() - .filter(streamId -> systemStream.getStream().equals(getPhysicalName(streamId))).collect(Collectors.toList()); - if (streamIds.size() > 1) { - throw new IllegalStateException(String.format("There was more than one stream found for system stream %s", systemStream)); - } + return streamIds.isEmpty() ? null : streamIds.get(0); + } - return streamIds.isEmpty() ? null : streamIds.get(0); + /** + * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza + * interacts with the stream, as opposed to a property of the stream itself. + *

+ * Note, because the translation is not perfect between SystemStream and streamId, + * this method is not identical to getProperty(streamId, property) + * + * @param systemStream the SystemStream for which the property value will be retrieved. + * @param property the samza property name excluding the leading delimiter. e.g. "samza.x.y" + */ + protected String getSamzaProperty(SystemStream systemStream, String property) { + if (!property.startsWith(SAMZA_PROPERTY)) { + throw new IllegalArgumentException( + String.format("Attempt to fetch a non samza property for SystemStream %s named %s", systemStream, property)); } - /** - * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza - * interacts with the stream, as opposed to a property of the stream itself. - *

- * Note, because the translation is not perfect between SystemStream and streamId, - * this method is not identical to getProperty(streamId, property) - * - * @param systemStream the SystemStream for which the property value will be retrieved. - * @param property the samza property name excluding the leading delimiter. e.g. "samza.x.y" - */ - protected String getSamzaProperty(SystemStream systemStream, String property) { - if (!property.startsWith(SAMZA_PROPERTY)) { - throw new IllegalArgumentException( - String.format("Attempt to fetch a non samza property for SystemStream %s named %s", systemStream, property)); - } - - String streamVal = getAllStreamProperties(systemStreamToStreamId(systemStream)).get(property); - return (streamVal != null) ? streamVal : getSystemStreamProperties(systemStream.getSystem(), systemStream.getStream()).get(property); - } + String streamVal = getAllStreamProperties(systemStreamToStreamId(systemStream)).get(property); + return (streamVal != null) ? streamVal : getSystemStreamProperties(systemStream.getSystem(), systemStream.getStream()).get(property); + } - /** - * Gets the System associated with the specified streamId. - * It first looks for the property - * streams.{streamId}.system - *

- * If no value was provided, it uses - * job.default.system - * - * @param streamId the identifier for the stream in the config. - * @return the system name associated with the stream. - */ - public String getSystem(String streamId) { - return getOrDefault(String.format(SYSTEM_FOR_STREAM_ID, streamId), - new JobConfig(this).getDefaultSystem().orElse(null)); - } + /** + * Gets the System associated with the specified streamId. + * It first looks for the property + * streams.{streamId}.system + *

+ * If no value was provided, it uses + * job.default.system + * + * @param streamId the identifier for the stream in the config. + * @return the system name associated with the stream. + */ + public String getSystem(String streamId) { + return getOrDefault(String.format(SYSTEM_FOR_STREAM_ID, streamId), + new JobConfig(this).getDefaultSystem().orElse(null)); + } - /** - * Gets the physical name for the specified streamId. - * - * @param streamId the identifier for the stream in the config. - * @return the physical identifier for the stream or the default if it is undefined. - */ - public String getPhysicalName(String streamId) { - // use streamId as the default physical name - return get(String.format(PHYSICAL_NAME_FOR_STREAM_ID, streamId, streamId)); - } + /** + * Gets the physical name for the specified streamId. + * + * @param streamId the identifier for the stream in the config. + * @return the physical identifier for the stream or the default if it is undefined. + */ + public String getPhysicalName(String streamId) { + // use streamId as the default physical name + return getOrDefault(String.format(PHYSICAL_NAME_FOR_STREAM_ID, streamId), streamId); + } - public Optional getStreamMsgSerde(SystemStream systemStream) { - return nonEmptyOption(getSamzaProperty(systemStream, MSG_SERDE)); - } + public Optional getStreamMsgSerde(SystemStream systemStream) { + return nonEmptyOption(getSamzaProperty(systemStream, MSG_SERDE)); + } - public Optional getStreamKeySerde(SystemStream systemStream) { - return nonEmptyOption(getSamzaProperty(systemStream, KEY_SERDE)); - } + public Optional getStreamKeySerde(SystemStream systemStream) { + return nonEmptyOption(getSamzaProperty(systemStream, KEY_SERDE)); + } - public boolean getResetOffset(SystemStream systemStream) { - String resetOffset = getSamzaProperty(systemStream, CONSUMER_RESET_OFFSET); - if (!resetOffset.equalsIgnoreCase("true") && !resetOffset.equalsIgnoreCase("false")) { - LOG.warn("Got a .samza.reset.offset configuration for SystemStream {} that is not true or false (was {})." + - " Defaulting to false.", systemStream, resetOffset); + public boolean getResetOffset(SystemStream systemStream) { + String resetOffset = getSamzaProperty(systemStream, CONSUMER_RESET_OFFSET, "false"); + if (!resetOffset.equalsIgnoreCase("true") && !resetOffset.equalsIgnoreCase("false")) { + LOG.warn("Got a .samza.reset.offset configuration for SystemStream {} that is not true or false (was {})." + + " Defaulting to false.", systemStream, resetOffset); - resetOffset = "false"; - } - return Boolean.valueOf(resetOffset); + resetOffset = "false"; } + return Boolean.valueOf(resetOffset); + } - /** - * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza - * interacts with the stream, as opposed to a property of the stream itself. - *

- * Note, because the translation is not perfect between SystemStream and streamId, - * this method is not identical to getProperty(streamId, property) - * - * @param systemStream the SystemStream for which the property value will be retrieved. - * @param property the samza property name excluding the leading delimiter. e.g. "samza.x.y" - * @param defaultValue the default value to use if the property value is not found - */ - protected String getSamzaProperty(SystemStream systemStream, String property, String defaultValue) { - String streamVal = getSamzaProperty(systemStream, property); - - return streamVal != null ? streamVal : defaultValue; - } + /** + * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza + * interacts with the stream, as opposed to a property of the stream itself. + *

+ * Note, because the translation is not perfect between SystemStream and streamId, + * this method is not identical to getProperty(streamId, property) + * + * @param systemStream the SystemStream for which the property value will be retrieved. + * @param property the samza property name excluding the leading delimiter. e.g. "samza.x.y" + * @param defaultValue the default value to use if the property value is not found + */ + protected String getSamzaProperty(SystemStream systemStream, String property, String defaultValue) { + String streamVal = getSamzaProperty(systemStream, property); - /** - * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza - * interacts with the stream, as opposed to a property of the stream itself. - *

- * Note, because the translation is not perfect between SystemStream and streamId, - * this method is not identical to getProperty(streamId, property) - */ - protected boolean containsSamzaProperty(SystemStream systemStream, String property) { - if (!property.startsWith(SAMZA_PROPERTY)) { - throw new IllegalArgumentException( - String.format("Attempt to fetch a non samza property for SystemStream %s named %s", systemStream, property)); - } - return getSamzaProperty(systemStream, property) != null; - } + return streamVal != null ? streamVal : defaultValue; + } - public boolean isResetOffsetConfigured(SystemStream systemStream) { - return containsSamzaProperty(systemStream, CONSUMER_RESET_OFFSET); + /** + * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza + * interacts with the stream, as opposed to a property of the stream itself. + *

+ * Note, because the translation is not perfect between SystemStream and streamId, + * this method is not identical to getProperty(streamId, property) + */ + protected boolean containsSamzaProperty(SystemStream systemStream, String property) { + if (!property.startsWith(SAMZA_PROPERTY)) { + throw new IllegalArgumentException( + String.format("Attempt to fetch a non samza property for SystemStream %s named %s", systemStream, property)); } + return getSamzaProperty(systemStream, property) != null; + } - public String getDefaultStreamOffset(SystemStream systemStream) { - return getSamzaProperty(systemStream, CONSUMER_OFFSET_DEFAULT); - } + public boolean isResetOffsetConfigured(SystemStream systemStream) { + return containsSamzaProperty(systemStream, CONSUMER_RESET_OFFSET); + } - public boolean isDefaultStreamOffsetConfigured(SystemStream systemStream) { - return containsSamzaProperty(systemStream, CONSUMER_OFFSET_DEFAULT); - } + public Optional getDefaultStreamOffset(SystemStream systemStream) { + return Optional.ofNullable(getSamzaProperty(systemStream, CONSUMER_OFFSET_DEFAULT)); + } - public boolean getBootstrapEnabled(SystemStream systemStream) { - return Boolean.parseBoolean(getSamzaProperty(systemStream, BOOTSTRAP)); - } + public boolean isDefaultStreamOffsetConfigured(SystemStream systemStream) { + return containsSamzaProperty(systemStream, CONSUMER_OFFSET_DEFAULT); + } - public boolean getBroadcastEnabled(SystemStream systemStream) { - return java.lang.Boolean.parseBoolean(getSamzaProperty(systemStream, BROADCAST)); - } + public boolean getBootstrapEnabled(SystemStream systemStream) { + return Boolean.parseBoolean(getSamzaProperty(systemStream, BOOTSTRAP)); + } - public int getPriority(SystemStream systemStream) { - return Integer.parseInt(getSamzaProperty(systemStream, PRIORITY, "-1")); - } + public boolean getBroadcastEnabled(SystemStream systemStream) { + return java.lang.Boolean.parseBoolean(getSamzaProperty(systemStream, BROADCAST)); + } - /** - * A streamId is translated to a SystemStream by looking up its System and physicalName. It - * will use the streamId as the stream name if the physicalName doesn't exist. - */ - public SystemStream streamIdToSystemStream(String streamId) { - return new SystemStream(getSystem(streamId), getPhysicalName(streamId)); - } + public int getPriority(SystemStream systemStream) { + return Integer.parseInt(getSamzaProperty(systemStream, PRIORITY, "-1")); + } + + /** + * A streamId is translated to a SystemStream by looking up its System and physicalName. It + * will use the streamId as the stream name if the physicalName doesn't exist. + */ + public SystemStream streamIdToSystemStream(String streamId) { + return new SystemStream(getSystem(streamId), getPhysicalName(streamId)); + } - /** - * Returns a list of all SystemStreams that have a serde defined from the config file. - */ - public Set getSerdeStreams(String systemName) { - Config subConf = subset(String.format("systems.%s.streams.", systemName, true)); - Set legacySystemStreams = subConf.keySet().stream() - .filter(k -> k.endsWith(MSG_SERDE) || k.endsWith(KEY_SERDE)) - .map(k -> { + /** + * Returns a list of all SystemStreams that have a serde defined from the config file. + */ + public Set getSerdeStreams(String systemName) { + Config subConf = subset(String.format("systems.%s.streams.", systemName, true)); + Set legacySystemStreams = subConf.keySet().stream() + .filter(k -> k.endsWith(MSG_SERDE) || k.endsWith(KEY_SERDE)) + .map(k -> { String streamName = k.substring(0, k.length() - 16 /* .samza.XXX.serde length */); return new SystemStream(systemName, streamName); }) - .collect(Collectors.toSet()); + .collect(Collectors.toSet()); - Set systemStreams = subset(STREAMS_PREFIX).keySet().stream() - .filter(k -> k.endsWith(MSG_SERDE) || k.endsWith(KEY_SERDE)) - .map(k -> k.substring(0, k.length() - 16 /* .samza.XXX.serde length */)) - .filter(streamId -> systemName.equals(getSystem(streamId))) - .map(streamId -> streamIdToSystemStream(streamId)).collect(Collectors.toSet()); + Set systemStreams = subset(STREAMS_PREFIX).keySet().stream() + .filter(k -> k.endsWith(MSG_SERDE) || k.endsWith(KEY_SERDE)) + .map(k -> k.substring(0, k.length() - 16 /* .samza.XXX.serde length */)) + .filter(streamId -> systemName.equals(getSystem(streamId))) + .map(streamId -> streamIdToSystemStream(streamId)).collect(Collectors.toSet()); - return Sets.union(legacySystemStreams, systemStreams).immutableCopy(); - } + return Sets.union(legacySystemStreams, systemStreams).immutableCopy(); + } - /** - * Gets the properties for the specified streamId from the config. - * It first applies any legacy configs from this config location: - * systems.{system}.streams.{stream}.* - *

- * It then overrides them with properties of the new config format: - * streams.{streamId}.* - *

- * Only returns properties of the stream itself, not any of the samza properties for the stream. - * - * @param streamId the identifier for the stream in the config. - * @return the merged map of config properties from both the legacy and new config styles - */ - public Config getStreamProperties(String streamId) { - MapConfig allProperties = getAllStreamProperties(streamId); - Config samzaProperties = allProperties.subset(SAMZA_PROPERTY, false); - Map filteredStreamProperties = - allProperties.entrySet().stream().filter(kv -> !samzaProperties.containsKey(kv.getKey())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - return new MapConfig(filteredStreamProperties); - } + /** + * Gets the properties for the specified streamId from the config. + * It first applies any legacy configs from this config location: + * systems.{system}.streams.{stream}.* + *

+ * It then overrides them with properties of the new config format: + * streams.{streamId}.* + *

+ * Only returns properties of the stream itself, not any of the samza properties for the stream. + * + * @param streamId the identifier for the stream in the config. + * @return the merged map of config properties from both the legacy and new config styles + */ + public Config getStreamProperties(String streamId) { + MapConfig allProperties = getAllStreamProperties(streamId); + Config samzaProperties = allProperties.subset(SAMZA_PROPERTY, false); + Map filteredStreamProperties = + allProperties.entrySet().stream().filter(kv -> !samzaProperties.containsKey(kv.getKey())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return new MapConfig(filteredStreamProperties); + } - /** - * Gets the boolean flag of whether the specified streamId is an intermediate stream - * - * @param streamId the identifier for the stream in the config. - * @return true if the stream is intermediate - */ - public boolean getIsIntermediateStream(String streamId) { - return getBoolean(String.format(IS_INTERMEDIATE_FOR_STREAM_ID, streamId), false); - } - - /** - * Gets the boolean flag of whether the committed messages specified streamId can be deleted - * - * @param streamId the identifier for the stream in the config. - * @return true if the committed messages of the stream can be deleted - */ - public boolean getDeleteCommittedMessages(String streamId) { - return getBoolean(String.format(DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, streamId), false); - } + /** + * Gets the boolean flag of whether the specified streamId is an intermediate stream + * + * @param streamId the identifier for the stream in the config. + * @return true if the stream is intermediate + */ + public boolean getIsIntermediateStream(String streamId) { + return getBoolean(String.format(IS_INTERMEDIATE_FOR_STREAM_ID, streamId), false); + } - public boolean getIsBounded(String streamId) { - return getBoolean(String.format(IS_BOUNDED_FOR_STREAM_ID, streamId), false); - } + /** + * Gets the boolean flag of whether the committed messages specified streamId can be deleted + * + * @param streamId the identifier for the stream in the config. + * @return true if the committed messages of the stream can be deleted + */ + public boolean getDeleteCommittedMessages(String streamId) { + return getBoolean(String.format(DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, streamId), false); + } + public boolean getIsBounded(String streamId) { + return getBoolean(String.format(IS_BOUNDED_FOR_STREAM_ID, streamId), false); } +} diff --git a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala index 1ffd111796..a21dfb6136 100644 --- a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala +++ b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala @@ -25,13 +25,14 @@ import java.net.{URL, UnknownHostException} import java.nio.file.Path import java.time.Duration import java.util -import java.util.Base64 +import java.util.{Base64, Optional} import java.util.concurrent.{ExecutorService, Executors, ScheduledExecutorService, TimeUnit} +import java.util.stream.Collectors import com.google.common.annotations.VisibleForTesting import com.google.common.util.concurrent.ThreadFactoryBuilder import org.apache.samza.checkpoint.{CheckpointListener, OffsetManager, OffsetManagerMetrics} -import org.apache.samza.config.StreamConfig1.Config2Stream +import org.apache.samza.config.StreamConfigJava import org.apache.samza.config._ import org.apache.samza.container.disk.DiskSpaceMonitor.Listener import org.apache.samza.container.disk.{DiskQuotaPolicyFactory, DiskSpaceMonitor, NoThrottlingDiskQuotaPolicyFactory, PollingScanDiskSpaceMonitor} @@ -303,9 +304,9 @@ object SamzaContainer extends Logging { * A Helper function to build a Map[SystemStream, Serde] for streams defined in the config. * This is useful to build both key and message serde maps. */ - val buildSystemStreamSerdeMap = (getSerdeName: (SystemStream) => Option[String]) => { + val buildSystemStreamSerdeMap = (getSerdeName: (SystemStream) => Optional[String]) => { (serdeStreams ++ inputSystemStreamPartitions) - .filter(systemStream => getSerdeName(systemStream).isDefined) + .filter(systemStream => getSerdeName(systemStream).isPresent) .flatMap(systemStream => { val serdeName = getSerdeName(systemStream).get val serde = serdes.getOrElse(serdeName, @@ -362,15 +363,16 @@ object SamzaContainer extends Logging { getChangelogSSPsForContainer(containerModel, changeLogSystemStreams).asJava) val intermediateStreams = streamConfig - .getStreamIds - .filter(streamConfig.getIsIntermediateStream(_)) + .getStreamIds() + .asScala + .filter((streamId:String) => streamConfig.getIsIntermediateStream(streamId)) .toList info("Got intermediate streams: %s" format intermediateStreams) val controlMessageKeySerdes = intermediateStreams .flatMap(streamId => { - val systemStream = config.streamIdToSystemStream(streamId) + val systemStream = streamConfig.streamIdToSystemStream(streamId) systemStreamKeySerdes.get(systemStream) .orElse(systemKeySerdes.get(systemStream.getSystem)) .map(serde => (systemStream, new StringSerde("UTF-8"))) @@ -378,7 +380,7 @@ object SamzaContainer extends Logging { val intermediateStreamMessageSerdes = intermediateStreams .flatMap(streamId => { - val systemStream = config.streamIdToSystemStream(streamId) + val systemStream = streamConfig.streamIdToSystemStream(streamId) systemStreamMessageSerdes.get(systemStream) .orElse(systemMessageSerdes.get(systemStream.getSystem)) .map(serde => (systemStream, new IntermediateMessageSerde(serde))) diff --git a/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala b/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala index 580475ca28..58c1848b66 100644 --- a/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala +++ b/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala @@ -22,10 +22,10 @@ package org.apache.samza.container import java.util.{Objects, Optional} import java.util.concurrent.ScheduledExecutorService + import org.apache.samza.SamzaException import org.apache.samza.checkpoint.OffsetManager -import org.apache.samza.config.Config -import org.apache.samza.config.StreamConfig1.Config2Stream +import org.apache.samza.config.{Config, StreamConfigJava} import org.apache.samza.context._ import org.apache.samza.job.model.{JobModel, TaskModel} import org.apache.samza.scheduler.{CallbackSchedulerImpl, EpochTimeScheduler, ScheduledCallback} diff --git a/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala b/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala index e8bdccfac4..7ead386bc5 100644 --- a/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala +++ b/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala @@ -22,7 +22,7 @@ package org.apache.samza.metrics.reporter import org.apache.samza.util.{Logging, StreamUtil, Util} import org.apache.samza.SamzaException import org.apache.samza.config.{Config, JobConfig, MetricsConfig, SerializerConfig, StreamConfigJava, SystemConfig} -import org.apache.samza.config.StreamConfig1.Config2Stream +import org.apache.samza.config.StreamConfigJava import org.apache.samza.metrics.MetricsReporter import org.apache.samza.metrics.MetricsReporterFactory import org.apache.samza.metrics.MetricsRegistryMap diff --git a/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java b/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java index 653d5b312b..e96cdbe332 100644 --- a/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java +++ b/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java @@ -18,14 +18,14 @@ */ package org.apache.samza.config; -import java.util.Collections; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import org.apache.samza.system.SystemStream; import org.junit.Test; -import scala.Option; -import scala.collection.JavaConverters; + +import java.util.Collections; +import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -48,11 +48,11 @@ public class TestStreamConfig { public void testGetStreamMsgSerde() { String value = "my.msg.serde"; doTestSamzaProperty(StreamConfigJava.MSG_SERDE, value, - (config, systemStream) -> assertEquals(Option.apply(value), config.getStreamMsgSerde(systemStream))); + (config, systemStream) -> assertEquals(Optional.of(value), config.getStreamMsgSerde(systemStream))); doTestSamzaProperty(StreamConfigJava.MSG_SERDE, "", - (config, systemStream) -> assertEquals(Option.empty(), config.getStreamMsgSerde(systemStream))); + (config, systemStream) -> assertEquals(Optional.empty(), config.getStreamMsgSerde(systemStream))); doTestSamzaPropertyDoesNotExist(StreamConfigJava.MSG_SERDE, - (config, systemStream) -> assertEquals(Option.empty(), config.getStreamMsgSerde(systemStream))); + (config, systemStream) -> assertEquals(Optional.empty(), config.getStreamMsgSerde(systemStream))); doTestSamzaPropertyInvalidConfig(StreamConfigJava::getStreamMsgSerde); } @@ -60,11 +60,11 @@ public void testGetStreamMsgSerde() { public void testGetStreamKeySerde() { String value = "my.key.serde"; doTestSamzaProperty(StreamConfigJava.KEY_SERDE, value, - (config, systemStream) -> assertEquals(Option.apply(value), config.getStreamKeySerde(systemStream))); + (config, systemStream) -> assertEquals(Optional.of(value), config.getStreamKeySerde(systemStream))); doTestSamzaProperty(StreamConfigJava.KEY_SERDE, "", - (config, systemStream) -> assertEquals(Option.empty(), config.getStreamKeySerde(systemStream))); + (config, systemStream) -> assertEquals(Optional.empty(), config.getStreamKeySerde(systemStream))); doTestSamzaPropertyDoesNotExist(StreamConfigJava.KEY_SERDE, - (config, systemStream) -> assertEquals(Option.empty(), config.getStreamKeySerde(systemStream))); + (config, systemStream) -> assertEquals(Optional.empty(), config.getStreamKeySerde(systemStream))); doTestSamzaPropertyInvalidConfig(StreamConfigJava::getStreamKeySerde); } @@ -100,11 +100,11 @@ public void testIsResetOffsetConfigured() { public void testGetDefaultStreamOffset() { String value = "my_offset_default"; doTestSamzaProperty(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, value, - (config, systemStream) -> assertEquals(Option.apply(value), config.getDefaultStreamOffset(systemStream))); + (config, systemStream) -> assertEquals(Optional.of(value), config.getDefaultStreamOffset(systemStream))); doTestSamzaProperty(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, "", - (config, systemStream) -> assertEquals(Option.apply(""), config.getDefaultStreamOffset(systemStream))); + (config, systemStream) -> assertEquals(Optional.of(""), config.getDefaultStreamOffset(systemStream))); doTestSamzaPropertyDoesNotExist(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, - (config, systemStream) -> assertEquals(Option.empty(), + (config, systemStream) -> assertEquals(Optional.empty(), new StreamConfigJava(config).getDefaultStreamOffset(systemStream))); doTestSamzaPropertyInvalidConfig(StreamConfigJava::getDefaultStreamOffset); } @@ -181,7 +181,7 @@ public void testGetSerdeStreams() { // not key/msg serde property for "systems..streams." streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); - assertEquals(Collections.emptySet(),streamConfig.getSerdeStreams(SYSTEM)); + assertEquals(Collections.emptySet(), streamConfig.getSerdeStreams(SYSTEM)); // not matching system for "systems..streams." streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( @@ -193,7 +193,7 @@ public void testGetSerdeStreams() { streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.KEY_SERDE, UNUSED_VALUE, String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), "otherSystem"))); - assertEquals(Collections.emptySet(),streamConfig.getSerdeStreams(SYSTEM)); + assertEquals(Collections.emptySet(), streamConfig.getSerdeStreams(SYSTEM)); String serdeValue = "my.serde.class"; diff --git a/samza-core/src/test/java/org/apache/samza/execution/TestExecutionPlanner.java b/samza-core/src/test/java/org/apache/samza/execution/TestExecutionPlanner.java index be7b15b20c..7c721839d7 100644 --- a/samza-core/src/test/java/org/apache/samza/execution/TestExecutionPlanner.java +++ b/samza-core/src/test/java/org/apache/samza/execution/TestExecutionPlanner.java @@ -40,7 +40,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig1$; +import org.apache.samza.config.StreamConfigJava; import org.apache.samza.config.TaskConfig; import org.apache.samza.serializers.StringSerde; import org.apache.samza.system.descriptors.GenericInputDescriptor; @@ -640,7 +640,7 @@ public void testDefaultPartitions() { @Test public void testBroadcastConfig() { Map map = new HashMap<>(config); - map.put(String.format(StreamConfig1$.MODULE$.BROADCAST_FOR_STREAM_ID(), "input1"), "true"); + map.put(String.format(StreamConfigJava.BROADCAST_FOR_STREAM_ID, "input1"), "true"); Config cfg = new MapConfig(map); ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager); diff --git a/samza-core/src/test/java/org/apache/samza/testUtils/StreamTestUtils.java b/samza-core/src/test/java/org/apache/samza/testUtils/StreamTestUtils.java index fe710938d4..976ccf57b1 100644 --- a/samza-core/src/test/java/org/apache/samza/testUtils/StreamTestUtils.java +++ b/samza-core/src/test/java/org/apache/samza/testUtils/StreamTestUtils.java @@ -19,7 +19,7 @@ package org.apache.samza.testUtils; import java.util.Map; -import org.apache.samza.config.StreamConfig1$; +import org.apache.samza.config.StreamConfigJava; public class StreamTestUtils { @@ -33,7 +33,7 @@ public class StreamTestUtils { */ public static void addStreamConfigs(Map configs, String streamId, String systemName, String physicalName) { - configs.put(String.format(StreamConfig1$.MODULE$.SYSTEM_FOR_STREAM_ID(), streamId), systemName); - configs.put(String.format(StreamConfig1$.MODULE$.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), physicalName); + configs.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, streamId), systemName); + configs.put(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, streamId), physicalName); } } \ No newline at end of file diff --git a/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala b/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala index 993f0e4225..1261d055b5 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala @@ -292,7 +292,8 @@ class KafkaConfig(config: Config) extends ScalaMapConfig(config) { // The method returns a map of storenames to changelog topic names, which are configured to use kafka as the changelog stream def getKafkaChangelogEnabledStores() = { val changelogConfigs = config.regexSubset(KafkaConfig.CHANGELOG_STREAM_NAMES_REGEX).asScala - var storeToChangelog = Map[String, String]() + //var storeToChangelog = Map[String, String]() + var storeToChangelog :util.HashMap[String, String] = new util.HashMap(); val storageConfig = new StorageConfig(config) val pattern = Pattern.compile(KafkaConfig.CHANGELOG_STREAM_NAMES_REGEX) @@ -304,7 +305,7 @@ class KafkaConfig(config: Config) extends ScalaMapConfig(config) { JavaOptionals.toRichOptional(storageConfig.getChangelogStream(storeName)).toOption.foreach(changelogName => { val systemStream = StreamUtil.getSystemStreamFromNames(changelogName) - storeToChangelog += storeName -> systemStream.getStream + storeToChangelog.put(storeName, systemStream.getStream) }) } storeToChangelog diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaConsumerProxy.java b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaConsumerProxy.java index 88f8510a7f..aedf3f1607 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaConsumerProxy.java +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaConsumerProxy.java @@ -99,6 +99,8 @@ public KafkaConsumerProxy(Consumer kafkaConsumer, String systemName, Strin /** * Add new partition to the list of polled partitions. * Bust only be called before {@link KafkaConsumerProxy#start} is called.. + * @param ssp - SystemStreamPartition to add + * @param nextOffset - add partition with this starting offset */ public void addTopicPartition(SystemStreamPartition ssp, long nextOffset) { LOG.info(String.format("Adding new topicPartition %s with offset %s to queue for consumer %s", ssp, nextOffset, @@ -339,6 +341,8 @@ protected IncomingMessageEnvelope handleNewRecord(ConsumerRecord consumerR /** * Protected to help extensions of this class build {@link IncomingMessageEnvelope}s. + * @param r consumer record to size + * @return the size of the serialized record */ protected int getRecordSize(ConsumerRecord r) { int keySize = (r.key() == null) ? 0 : r.serializedKeySize(); diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala index 52d6354584..b3207f24b6 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala @@ -28,6 +28,7 @@ import org.apache.samza.config.KafkaConfig.Config2Kafka import org.apache.samza.config._ import org.apache.samza.metrics.MetricsRegistry import org.apache.samza.system.{SystemAdmin, SystemConsumer, SystemFactory, SystemProducer} +import scala.collection.JavaConverters._ import org.apache.samza.util._ object KafkaSystemFactory extends Logging { @@ -109,7 +110,7 @@ class KafkaSystemFactory extends SystemFactory with Logging { val appConfig = new ApplicationConfig(config) if (appConfig.getAppMode == ApplicationMode.BATCH) { val streamConfig = new StreamConfigJava(config) - streamConfig.getStreamIds().filter(streamConfig.getIsIntermediateStream(_)).map(streamId => { + streamConfig.getStreamIds().asScala.filter(streamConfig.getIsIntermediateStream(_)).map(streamId => { // only the override here val properties = new Properties() properties.putIfAbsent("retention.ms", String.valueOf(KafkaConfig.DEFAULT_RETENTION_MS_FOR_BATCH)) diff --git a/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala b/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala index d94f414b32..ea6c3f81e7 100644 --- a/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala +++ b/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala @@ -102,17 +102,17 @@ class TestKafkaConfig { assertEquals(kafkaConfig.getChangelogKafkaProperties("test2").getProperty("max.message.bytes"), "1024000") assertEquals(kafkaConfig.getChangelogKafkaProperties("test3").getProperty("cleanup.policy"), "compact") val storeToChangelog = kafkaConfig.getKafkaChangelogEnabledStores() - assertEquals("mychangelog1", storeToChangelog.get("test1").getOrElse("")) - assertEquals("mychangelog2", storeToChangelog.get("test2").getOrElse("")) - assertEquals("otherstream", storeToChangelog.get("test3").getOrElse("")) + assertEquals("mychangelog1", storeToChangelog.getOrDefault("test1", "")) + assertEquals("mychangelog2", storeToChangelog.getOrDefault("test2", "")) + assertEquals("otherstream", storeToChangelog.getOrDefault("test3", "")) assertNull(kafkaConfig.getChangelogKafkaProperties("test1").getProperty("retention.ms")) assertNull(kafkaConfig.getChangelogKafkaProperties("test2").getProperty("retention.ms")) props.setProperty("systems." + SYSTEM_NAME + ".samza.factory", "org.apache.samza.system.kafka.SomeOtherFactory") val storeToChangelog1 = kafkaConfig.getKafkaChangelogEnabledStores() - assertEquals("mychangelog1", storeToChangelog1.get("test1").getOrElse("")) - assertEquals("mychangelog2", storeToChangelog1.get("test2").getOrElse("")) - assertEquals("otherstream", storeToChangelog1.get("test3").getOrElse("")) + assertEquals("mychangelog1", storeToChangelog1.getOrDefault("test1", "")) + assertEquals("mychangelog2", storeToChangelog1.getOrDefault("test2", "")) + assertEquals("otherstream", storeToChangelog1.getOrDefault("test3", "")) assertEquals(kafkaConfig.getChangelogKafkaProperties("test4").getProperty("cleanup.policy"), "delete") assertEquals(kafkaConfig.getChangelogKafkaProperties("test4").getProperty("retention.ms"), "3600") diff --git a/samza-sql/src/main/java/org/apache/samza/sql/SamzaSqlInputMessage.java b/samza-sql/src/main/java/org/apache/samza/sql/SamzaSqlInputMessage.java index ad6706cb06..7ce03397cd 100644 --- a/samza-sql/src/main/java/org/apache/samza/sql/SamzaSqlInputMessage.java +++ b/samza-sql/src/main/java/org/apache/samza/sql/SamzaSqlInputMessage.java @@ -43,9 +43,9 @@ private SamzaSqlInputMessage(KV keyAndMessageKV, SamzaSqlRelMsgM /** * Constructs a new SamzaSqMessage given the input arguments - * @param keyAndMessageKV - * @param metadata - * @return + * @param keyAndMessageKV key-value of the message + * @param metadata metadata of the message + * @return new object of SamzaSqlInputMessage type */ public static SamzaSqlInputMessage of (KV keyAndMessageKV, SamzaSqlRelMsgMetadata metadata) { return new SamzaSqlInputMessage(keyAndMessageKV, metadata); diff --git a/samza-sql/src/main/java/org/apache/samza/sql/planner/SamzaSqlValidator.java b/samza-sql/src/main/java/org/apache/samza/sql/planner/SamzaSqlValidator.java index 9482a75d6c..97b5de93c4 100644 --- a/samza-sql/src/main/java/org/apache/samza/sql/planner/SamzaSqlValidator.java +++ b/samza-sql/src/main/java/org/apache/samza/sql/planner/SamzaSqlValidator.java @@ -63,7 +63,7 @@ public SamzaSqlValidator(Config config) { /** * Validate a list of sql statements * @param sqlStmts list of sql statements - * @throws SamzaSqlValidatorException + * @throws SamzaSqlValidatorException exception for sql validation */ public void validate(List sqlStmts) throws SamzaSqlValidatorException { SamzaSqlApplicationConfig sqlConfig = SamzaSqlDslConverter.getSqlConfig(sqlStmts, config); From a109588b6f49023bb1aca1e3634379f44ece0345 Mon Sep 17 00:00:00 2001 From: Boris S Date: Thu, 26 Sep 2019 16:32:51 -0700 Subject: [PATCH 6/9] add comments --- .../apache/samza/config/StreamConfigJava.java | 120 ++++++++++++------ 1 file changed, 84 insertions(+), 36 deletions(-) diff --git a/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java b/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java index 889bc4fbc6..3cf1e276a9 100644 --- a/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java +++ b/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java @@ -67,7 +67,23 @@ public class StreamConfigJava extends MapConfig { public static final String BOOTSTRAP_FOR_STREAM_ID = STREAM_ID_PREFIX + BOOTSTRAP; public static final String BROADCAST_FOR_STREAM_ID = STREAM_ID_PREFIX + BROADCAST; - //implicit def Config2Stream(config: Config) = new StreamConfig(config) + /** + * Helper for accessing configs related to stream properties. + * + * For most configs, this currently supports two different formats for specifying stream properties: + * 1) "streams.{streamId}.{property}" (recommended to use this format) + * 2) "systems.{systemName}.streams.{streamName}.{property}" (legacy) + * Note that some config lookups are only supported through the "streams.{streamId}.{property}". See the specific + * accessor method to determine which formats are supported. + * + * Summary of terms: + * - streamId: logical identifier used for a stream; configs are specified using this streamId + * - physical stream: concrete name for a stream (if the physical stream is not explicitly configured, then the streamId + * is used as the physical stream + * - streamName: within the javadoc for this class, streamName is the same as physical stream + * - samza property: property which is Samza-specific, which will have "samza." as a prefix (e.g. "samza.key.serde"); + * this is in contrast to stream-specific properties which are related to specific stream technologies + */ private Optional nonEmptyOption(String value) { if (value == null || value.isEmpty()) { @@ -83,12 +99,15 @@ public StreamConfigJava(Config config) { /** - * Gets the stream properties from the legacy config style: - * systems.{system}.streams.{streams}.* + * Finds the properties from the legacy config style (config key includes system). + * This will return a Config with the properties that match the following formats (if a property is specified through + * multiple formats, priority is top to bottom): + * 1) "systems.{systemName}.streams.{streamName}.{property}" + * 2) "systems.{systemName}.default.stream.{property}" * * @param systemName the system name under which the properties are configured * @param streamName the stream name - * @return the map of properties for the stream + * @return the map of properties for the stream */ private Map getSystemStreamProperties(String systemName, String streamName) { if (systemName == null) { @@ -102,15 +121,17 @@ private Map getSystemStreamProperties(String systemName, String /** - * Gets the properties for the specified streamId from the config. - * It first applies any legacy configs from this config location: - * systems.{system}.streams.{stream}.* - *

- * It then overrides them with properties of the new config format: - * streams.{streamId}.* + * Gets all of the properties for the specified streamId (includes current and legacy config styles). + * This will return a Config with the properties that match the following formats (if a property is specified through + * multiple formats, priority is top to bottom): + * 1) "streams.{streamId}.{property}" + * 2) "systems.{systemName}.streams.{streamName}.{property}" where systemName is the system mapped to the streamId in + * the config and streamName is the physical stream name mapped to the stream id + * 3) "systems.{systemName}.default.stream.{property}" where systemName is the system mapped to the streamId in the + * config * * @param streamId the identifier for the stream in the config. - * @return the merged map of config properties from both the legacy and new config styles + * @return the merged map of config properties from both the legacy and new config styles */ private MapConfig getAllStreamProperties(String streamId) { Config allProperties = subset(String.format(STREAM_ID_PREFIX, streamId)); @@ -133,6 +154,17 @@ private List getStreamIdsForSystem(String system) { return getStreamIds().stream().filter(streamId -> system.equals(getSystem(streamId))).collect(Collectors.toList()); } + /** + * Finds the stream id which corresponds to the systemStream. + * This finds the stream id that is mapped to the system in systemStream through the config and that has a physical + * name (the physical name might be the stream id itself if there is no explicit mapping) that matches the stream in + * systemStream. + * Note: If the stream in the systemStream is a stream id which is mapped to a physical stream, then that stream won't + * be returned as a stream id here, since the stream in systemStream doesn't match the physical stream name. + * + * @param systemStream system stream to map to stream id + * @return stream id corresponding to the system stream + */ public String systemStreamToStreamId(SystemStream systemStream) { List streamIds = getStreamIdsForSystem(systemStream.getSystem()).stream() .filter(streamId -> systemStream.getStream().equals(getPhysicalName(streamId))).collect(Collectors.toList()); @@ -146,12 +178,27 @@ public String systemStreamToStreamId(SystemStream systemStream) { /** * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza * interacts with the stream, as opposed to a property of the stream itself. - *

- * Note, because the translation is not perfect between SystemStream and streamId, - * this method is not identical to getProperty(streamId, property) + * + * First, tries to map the systemStream to a streamId. This will only find a streamId if the stream is a physical name + * (explicitly mapped physical name or a stream id without a physical name mapping). That means this will not map a + * stream id to itself if there is a mapping from the stream id to a physical stream name. This also requires that the + * stream id is mapped to a system in the config. + * If a stream id is found: + * 1) Look for "streams.{streamId}.{property}" for the stream id. + * 2) Otherwise, look for "systems.{systemName}.streams.{streamName}.{property}" in which the systemName is the system + * mapped to the stream id and the streamName is the physical stream name for the stream id. + * 3) Otherwise, look for "systems.{systemName}.default.stream.{property}" in which the systemName is the system + * mapped to the stream id. + * If a stream id was not found or no property could be found using the above keys: + * 1) Look for "systems.{systemName}.streams.{streamName}.{property}" in which the systemName is the system in the + * input systemStream and the streamName is the stream from the input systemStream. + * 2) Otherwise, look for "systems.{systemName}.default.stream.{property}" in which the systemName is the system + * in the input systemStream. * * @param systemStream the SystemStream for which the property value will be retrieved. - * @param property the samza property name excluding the leading delimiter. e.g. "samza.x.y" + * @param property the samza property key (including the "samza." prefix); for example, for both + * "streams.streamId.samza.prop.key" and "systems.system.streams.streamName.samza.prop.key", this + * argument should have the value "samza.prop.key" */ protected String getSamzaProperty(SystemStream systemStream, String property) { if (!property.startsWith(SAMZA_PROPERTY)) { @@ -212,14 +259,13 @@ public boolean getResetOffset(SystemStream systemStream) { /** - * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza - * interacts with the stream, as opposed to a property of the stream itself. - *

- * Note, because the translation is not perfect between SystemStream and streamId, - * this method is not identical to getProperty(streamId, property) + * Gets a Samza property, with a default value used if no property value is found. + * See getSamzaProperty(SystemStream, String). * * @param systemStream the SystemStream for which the property value will be retrieved. - * @param property the samza property name excluding the leading delimiter. e.g. "samza.x.y" + * @param property the samza property key (including the "samza." prefix); for example, for both + * "streams.streamId.samza.prop.key" and "systems.system.streams.streamName.samza.prop.key", this + * argument should have the value "samza.prop.key" * @param defaultValue the default value to use if the property value is not found */ protected String getSamzaProperty(SystemStream systemStream, String property, String defaultValue) { @@ -229,11 +275,13 @@ protected String getSamzaProperty(SystemStream systemStream, String property, St } /** - * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza - * interacts with the stream, as opposed to a property of the stream itself. - *

- * Note, because the translation is not perfect between SystemStream and streamId, - * this method is not identical to getProperty(streamId, property) + * Determines if a Samza property is specified. + * See getSamzaProperty(SystemStream, String). + * + * @param systemStream the SystemStream for the property value to check + * @param property the samza property key (including the "samza." prefix); for example, for both + * "streams.streamId.samza.prop.key" and "systems.system.streams.streamName.samza.prop.key", this + * argument should have the value "samza.prop.key" */ protected boolean containsSamzaProperty(SystemStream systemStream, String property) { if (!property.startsWith(SAMZA_PROPERTY)) { @@ -298,18 +346,18 @@ public Set getSerdeStreams(String systemName) { return Sets.union(legacySystemStreams, systemStreams).immutableCopy(); } - /** - * Gets the properties for the specified streamId from the config. - * It first applies any legacy configs from this config location: - * systems.{system}.streams.{stream}.* - *

- * It then overrides them with properties of the new config format: - * streams.{streamId}.* - *

- * Only returns properties of the stream itself, not any of the samza properties for the stream. + /* Gets the properties for the streamId which are not Samza properties (i.e. do not have a "samza." prefix). This + * includes current and legacy config styles. + * This will return a Config with the properties that match the following formats (if a property is specified through + * multiple formats, priority is top to bottom): + * 1) "streams.{streamId}.{property}" + * 2) "systems.{systemName}.streams.{streamName}.{property}" where systemName is the system mapped to the streamId in + * the config and streamName is the physical stream name mapped to the stream id + * 3) "systems.{systemName}.default.stream.{property}" where systemName is the system mapped to the streamId in the + * config * * @param streamId the identifier for the stream in the config. - * @return the merged map of config properties from both the legacy and new config styles + * @return the merged map of config properties from both the legacy and new config styles */ public Config getStreamProperties(String streamId) { MapConfig allProperties = getAllStreamProperties(streamId); From f0b2f4c3cc866629f824adc7b1b9e0239eeaced9 Mon Sep 17 00:00:00 2001 From: Boris S Date: Thu, 26 Sep 2019 16:36:06 -0700 Subject: [PATCH 7/9] rename back to StreamConfig --- .../system/kinesis/KinesisSystemFactory.java | 4 +- .../samza/system/eventhub/EventHubConfig.java | 4 +- .../producer/AsyncSystemProducer.java | 4 +- .../samza/config/DefaultChooserConfig.java | 4 +- .../org/apache/samza/config/SystemConfig.java | 4 +- .../samza/execution/ExecutionPlanner.java | 6 +- .../JobNodeConfigurationGenerator.java | 12 +- .../apache/samza/execution/JobPlanner.java | 4 +- .../apache/samza/execution/StreamEdge.java | 18 +- .../apache/samza/execution/StreamManager.java | 2 +- .../operators/impl/OperatorImplGraph.java | 10 +- .../samza/storage/StorageManagerUtil.java | 5 +- .../storage/TaskSideInputStorageManager.java | 5 +- .../org/apache/samza/util/StreamUtil.java | 8 +- .../samza/checkpoint/OffsetManager.scala | 4 +- ...treamConfigJava.java => StreamConfig.java} | 4 +- .../samza/container/SamzaContainer.scala | 5 +- .../apache/samza/container/TaskInstance.scala | 4 +- .../org/apache/samza/job/JobRunner.scala | 2 +- .../MetricsSnapshotReporterFactory.scala | 6 +- .../apache/samza/config/TestStreamConfig.java | 460 +++++++++--------- .../apache/samza/config/TestSystemConfig.java | 20 +- .../samza/execution/TestExecutionPlanner.java | 4 +- .../samza/execution/TestStreamEdge.java | 8 +- .../operators/impl/TestOperatorImplGraph.java | 4 +- .../samza/testUtils/StreamTestUtils.java | 6 +- .../org/apache/samza/util/TestStreamUtil.java | 120 ++--- .../samza/system/kafka/KafkaSystemAdmin.java | 7 +- .../system/kafka/KafkaSystemFactory.scala | 2 +- .../samza/config/Log4jSystemConfig.java | 2 +- .../samza/config/Log4jSystemConfig.java | 2 +- .../samza/sql/interfaces/SqlIOConfig.java | 8 +- .../sql/translator/TestQueryTranslator.java | 20 +- .../samza/test/framework/TestRunner.java | 8 +- .../samza/test/operator/TestAsyncFlatMap.java | 4 +- .../TestLocalTableWithSideInputsEndToEnd.java | 8 +- 36 files changed, 399 insertions(+), 399 deletions(-) rename samza-core/src/main/scala/org/apache/samza/config/{StreamConfigJava.java => StreamConfig.java} (99%) diff --git a/samza-aws/src/main/java/org/apache/samza/system/kinesis/KinesisSystemFactory.java b/samza-aws/src/main/java/org/apache/samza/system/kinesis/KinesisSystemFactory.java index 09349e41f9..27580220c4 100644 --- a/samza-aws/src/main/java/org/apache/samza/system/kinesis/KinesisSystemFactory.java +++ b/samza-aws/src/main/java/org/apache/samza/system/kinesis/KinesisSystemFactory.java @@ -22,7 +22,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.ConfigException; import org.apache.samza.config.JobConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.config.TaskConfig; import org.apache.samza.container.grouper.stream.AllSspToSingleTaskGrouperFactory; import org.apache.samza.metrics.MetricsRegistry; @@ -76,7 +76,7 @@ protected void validateConfig(String system, Config config) { // Kinesis streams cannot be configured as bootstrap streams KinesisConfig kConfig = new KinesisConfig(config); kConfig.getKinesisStreams(system).forEach(stream -> { - StreamConfigJava streamConfig = new StreamConfigJava(kConfig); + StreamConfig streamConfig = new StreamConfig(kConfig); SystemStream ss = new SystemStream(system, stream); if (streamConfig.getBootstrapEnabled(ss)) { throw new ConfigException("Kinesis streams cannot be configured as bootstrap streams."); diff --git a/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java b/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java index 2bd52d1d95..69a921e1a3 100644 --- a/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java +++ b/samza-azure/src/main/java/org/apache/samza/system/eventhub/EventHubConfig.java @@ -24,7 +24,7 @@ import org.apache.samza.SamzaException; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.system.eventhub.producer.EventHubSystemProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,7 +105,7 @@ public EventHubConfig(Config config) { super(config); // Build reverse index for streamName -> streamId - StreamConfigJava streamConfig = new StreamConfigJava(config); + StreamConfig streamConfig = new StreamConfig(config); LOG.info("Building mappings from physicalName to streamId"); streamConfig.getStreamIds() diff --git a/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/AsyncSystemProducer.java b/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/AsyncSystemProducer.java index 0355227b83..83d51ede92 100644 --- a/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/AsyncSystemProducer.java +++ b/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/AsyncSystemProducer.java @@ -34,7 +34,7 @@ import java.util.stream.Collectors; import org.apache.samza.SamzaException; import org.apache.samza.config.Config; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.metrics.Counter; import org.apache.samza.metrics.MetricsRegistry; import org.apache.samza.system.OutgoingMessageEnvelope; @@ -109,7 +109,7 @@ public abstract class AsyncSystemProducer implements SystemProducer { * @param metricsRegistry the registry */ public AsyncSystemProducer(String systemName, Config config, MetricsRegistry metricsRegistry) { - StreamConfigJava sconfig = new StreamConfigJava(config); + StreamConfig sconfig = new StreamConfig(config); streamIds = config.getList(String.format(CONFIG_STREAM_LIST, systemName)); physicalToStreamIds = streamIds.stream().collect(Collectors.toMap(sconfig::getPhysicalName, Function.identity())); diff --git a/samza-core/src/main/java/org/apache/samza/config/DefaultChooserConfig.java b/samza-core/src/main/java/org/apache/samza/config/DefaultChooserConfig.java index 1d6090750a..c753b4e10a 100644 --- a/samza-core/src/main/java/org/apache/samza/config/DefaultChooserConfig.java +++ b/samza-core/src/main/java/org/apache/samza/config/DefaultChooserConfig.java @@ -33,12 +33,12 @@ public class DefaultChooserConfig extends MapConfig { private static final String BATCH_SIZE = "task.consumer.batch.size"; private final TaskConfig taskConfig; - private final StreamConfigJava streamConfig; + private final StreamConfig streamConfig; public DefaultChooserConfig(Config config) { super(config); taskConfig = new TaskConfig(config); - streamConfig = new StreamConfigJava(config); + streamConfig = new StreamConfig(config); } /** diff --git a/samza-core/src/main/java/org/apache/samza/config/SystemConfig.java b/samza-core/src/main/java/org/apache/samza/config/SystemConfig.java index 74c4cf2396..bfbc2979e9 100644 --- a/samza-core/src/main/java/org/apache/samza/config/SystemConfig.java +++ b/samza-core/src/main/java/org/apache/samza/config/SystemConfig.java @@ -160,7 +160,7 @@ public String getSystemOffsetDefault(String systemName) { * @return the key serde for the {@code systemName}, or empty if it was not found */ public Optional getSystemKeySerde(String systemName) { - return getSystemDefaultStreamProperty(systemName, StreamConfigJava.KEY_SERDE); + return getSystemDefaultStreamProperty(systemName, StreamConfig.KEY_SERDE); } /** @@ -168,7 +168,7 @@ public Optional getSystemKeySerde(String systemName) { * @return the message serde for the {@code systemName}, or empty if it was not found */ public Optional getSystemMsgSerde(String systemName) { - return getSystemDefaultStreamProperty(systemName, StreamConfigJava.MSG_SERDE); + return getSystemDefaultStreamProperty(systemName, StreamConfig.MSG_SERDE); } /** diff --git a/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java b/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java index 3ea2b36528..d59eb514bb 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java +++ b/samza-core/src/main/java/org/apache/samza/execution/ExecutionPlanner.java @@ -44,7 +44,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.operators.spec.InputOperatorSpec; import org.apache.samza.operators.spec.OperatorSpec; import org.apache.samza.operators.spec.StreamTableJoinOperatorSpec; @@ -68,12 +68,12 @@ public class ExecutionPlanner { private final Config config; private final StreamManager streamManager; - private final StreamConfigJava streamConfig; + private final StreamConfig streamConfig; public ExecutionPlanner(Config config, StreamManager streamManager) { this.config = config; this.streamManager = streamManager; - this.streamConfig = new StreamConfigJava(config); + this.streamConfig = new StreamConfig(config); } public ExecutionPlan plan(ApplicationDescriptorImpl appDesc) { diff --git a/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java b/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java index 47eca30c8e..ca46d1c3e1 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java +++ b/samza-core/src/main/java/org/apache/samza/execution/JobNodeConfigurationGenerator.java @@ -39,7 +39,7 @@ import org.apache.samza.config.MapConfig; import org.apache.samza.config.SerializerConfig; import org.apache.samza.config.StorageConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.config.TaskConfig; import org.apache.samza.operators.KV; import org.apache.samza.operators.spec.JoinOperatorSpec; @@ -251,7 +251,7 @@ private void configureTables(Map generatedConfig, Config origina .map(sideInput -> StreamUtil.getSystemStreamFromNameOrId(originalConfig, sideInput)) .forEach(systemStream -> { inputs.add(StreamUtil.getNameFromSystemStream(systemStream)); - generatedConfig.put(String.format(StreamConfigJava.STREAM_PREFIX + StreamConfigJava.BOOTSTRAP, + generatedConfig.put(String.format(StreamConfig.STREAM_PREFIX + StreamConfig.BOOTSTRAP, systemStream.getSystem(), systemStream.getStream()), "true"); }); } @@ -314,14 +314,14 @@ private void configureSerdes(Map configs, Map { - String streamIdPrefix = String.format(StreamConfigJava.STREAM_ID_PREFIX, streamId); - String keySerdeConfigKey = streamIdPrefix + StreamConfigJava.KEY_SERDE; + String streamIdPrefix = String.format(StreamConfig.STREAM_ID_PREFIX, streamId); + String keySerdeConfigKey = streamIdPrefix + StreamConfig.KEY_SERDE; configs.put(keySerdeConfigKey, serdeUUIDs.get(serde)); }); streamMsgSerdes.forEach((streamId, serde) -> { - String streamIdPrefix = String.format(StreamConfigJava.STREAM_ID_PREFIX, streamId); - String valueSerdeConfigKey = streamIdPrefix + StreamConfigJava.MSG_SERDE; + String streamIdPrefix = String.format(StreamConfig.STREAM_ID_PREFIX, streamId); + String valueSerdeConfigKey = streamIdPrefix + StreamConfig.MSG_SERDE; configs.put(valueSerdeConfigKey, serdeUUIDs.get(serde)); }); diff --git a/samza-core/src/main/java/org/apache/samza/execution/JobPlanner.java b/samza-core/src/main/java/org/apache/samza/execution/JobPlanner.java index 3f4c79e78b..f30080456b 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/JobPlanner.java +++ b/samza-core/src/main/java/org/apache/samza/execution/JobPlanner.java @@ -35,7 +35,7 @@ import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; import org.apache.samza.config.ShellCommandConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.config.TaskConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,7 +126,7 @@ private Map getGeneratedConfig(String runId) { generatedConfig.put(ApplicationConfig.APP_RUN_ID, runId); } - StreamConfigJava streamConfig = new StreamConfigJava(userConfig); + StreamConfig streamConfig = new StreamConfig(userConfig); Set inputStreamIds = new HashSet<>(appDesc.getInputStreamIds()); inputStreamIds.removeAll(appDesc.getOutputStreamIds()); // exclude intermediate streams ApplicationConfig.ApplicationMode mode = diff --git a/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java b/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java index 915508f377..4999d06993 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java +++ b/samza-core/src/main/java/org/apache/samza/execution/StreamEdge.java @@ -22,7 +22,7 @@ import org.apache.samza.config.ApplicationConfig; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.system.StreamSpec; import org.apache.samza.system.SystemStream; import org.apache.samza.util.StreamUtil; @@ -56,7 +56,7 @@ public class StreamEdge { this.isIntermediate = isIntermediate; // broadcast can be configured either by an operator or via the configs this.isBroadcast = - isBroadcast || new StreamConfigJava(config).getBroadcastEnabled(streamSpec.toSystemStream()); + isBroadcast || new StreamConfig(config).getBroadcastEnabled(streamSpec.toSystemStream()); this.config = config; if (isBroadcast && isIntermediate) { partitions = 1; @@ -122,21 +122,21 @@ Config generateConfig() { Map streamConfig = new HashMap<>(); StreamSpec spec = getStreamSpec(); String streamId = spec.getId(); - streamConfig.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, streamId), spec.getSystemName()); - streamConfig.put(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, streamId), spec.getPhysicalName()); + streamConfig.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, streamId), spec.getSystemName()); + streamConfig.put(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, streamId), spec.getPhysicalName()); if (isIntermediate()) { - streamConfig.put(String.format(StreamConfigJava.IS_INTERMEDIATE_FOR_STREAM_ID, streamId), "true"); - streamConfig.put(String.format(StreamConfigJava.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, streamId), "true"); + streamConfig.put(String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID, streamId), "true"); + streamConfig.put(String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, streamId), "true"); // Setting offset.default to oldest only if the job is running in batch mode if (ApplicationConfig.ApplicationMode.BATCH.equals(new ApplicationConfig(config).getAppMode())) { - streamConfig.put(String.format(StreamConfigJava.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID, streamId), "oldest"); + streamConfig.put(String.format(StreamConfig.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID, streamId), "oldest"); } - streamConfig.put(String.format(StreamConfigJava.PRIORITY_FOR_STREAM_ID, streamId), String.valueOf(Integer.MAX_VALUE)); + streamConfig.put(String.format(StreamConfig.PRIORITY_FOR_STREAM_ID, streamId), String.valueOf(Integer.MAX_VALUE)); } spec.getConfig().forEach((property, value) -> { - streamConfig.put(String.format(StreamConfigJava.STREAM_ID_PREFIX, streamId) + property, value); + streamConfig.put(String.format(StreamConfig.STREAM_ID_PREFIX, streamId) + property, value); }); return new MapConfig(streamConfig); diff --git a/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java b/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java index 2dab3445b1..8d6cb01d71 100644 --- a/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java +++ b/samza-core/src/main/java/org/apache/samza/execution/StreamManager.java @@ -109,7 +109,7 @@ public void clearStreamsFromPreviousRun(Config prevConfig, ClassLoader classLoad ApplicationConfig appConfig = new ApplicationConfig(prevConfig); LOGGER.info("run.id from previous run is {}", appConfig.getRunId()); - StreamConfigJava streamConfig = new StreamConfigJava(prevConfig); + StreamConfig streamConfig = new StreamConfig(prevConfig); //Find all intermediate streams and clean up Set intStreams = streamConfig.getStreamIds().stream() diff --git a/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java b/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java index 86cd375083..4cf7201937 100644 --- a/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java +++ b/samza-core/src/main/java/org/apache/samza/operators/impl/OperatorImplGraph.java @@ -22,7 +22,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import org.apache.samza.config.Config; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.context.Context; import org.apache.samza.context.InternalTaskContext; import org.apache.samza.job.model.JobModel; @@ -99,7 +99,7 @@ public class OperatorImplGraph { */ public OperatorImplGraph(OperatorSpecGraph specGraph, Context context, Clock clock) { this.clock = clock; - StreamConfigJava streamConfig = new StreamConfigJava(context.getJobContext().getConfig()); + StreamConfig streamConfig = new StreamConfig(context.getJobContext().getConfig()); this.internalTaskContext = new InternalTaskContext(context); Map producerTaskCounts = hasIntermediateStreams(specGraph) @@ -215,7 +215,7 @@ private OperatorImpl createAndRegisterOperatorImpl(OperatorSpec prevOperatorSpec */ OperatorImpl createOperatorImpl(OperatorSpec prevOperatorSpec, OperatorSpec operatorSpec, Context context) { Config config = context.getJobContext().getConfig(); - StreamConfigJava streamConfig = new StreamConfigJava(config); + StreamConfig streamConfig = new StreamConfig(config); if (operatorSpec instanceof InputOperatorSpec) { return new InputOperatorImpl((InputOperatorSpec) operatorSpec); } else if (operatorSpec instanceof StreamOperatorSpec) { @@ -380,7 +380,7 @@ static Multimap getStreamToConsumerTasks(JobModel jobModel * @return mapping from output streams to input streams */ static Multimap getIntermediateToInputStreamsMap( - OperatorSpecGraph specGraph, StreamConfigJava streamConfig) { + OperatorSpecGraph specGraph, StreamConfig streamConfig) { Multimap outputToInputStreams = HashMultimap.create(); specGraph.getInputOperators().entrySet().stream() .forEach(entry -> { @@ -391,7 +391,7 @@ static Multimap getIntermediateToInputStreamsMap( } private static void computeOutputToInput(SystemStream input, OperatorSpec opSpec, - Multimap outputToInputStreams, StreamConfigJava streamConfig) { + Multimap outputToInputStreams, StreamConfig streamConfig) { if (opSpec instanceof PartitionByOperatorSpec) { PartitionByOperatorSpec spec = (PartitionByOperatorSpec) opSpec; SystemStream systemStream = streamConfig.streamIdToSystemStream(spec.getOutputStream().getStreamId()); diff --git a/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java b/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java index 09a3ff230c..9d82b0e66a 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java +++ b/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; import org.apache.samza.clustermanager.StandbyTaskUtil; +import org.apache.samza.config.StreamConfig; import org.apache.samza.container.TaskName; import org.apache.samza.job.model.TaskMode; import org.apache.samza.serializers.model.SamzaObjectMapper; @@ -57,8 +58,8 @@ public class StorageManagerUtil { /** * Fetch the starting offset for the input {@link SystemStreamPartition} * - * Note: The method doesn't respect {@link org.apache.samza.config.StreamConfigJava#CONSUMER_OFFSET_DEFAULT} and - * {@link org.apache.samza.config.StreamConfigJava#CONSUMER_RESET_OFFSET} configurations. It will use the locally + * Note: The method doesn't respect {@link StreamConfig#CONSUMER_OFFSET_DEFAULT} and + * {@link StreamConfig#CONSUMER_RESET_OFFSET} configurations. It will use the locally * checkpointed offset if it is valid, or fall back to oldest offset of the stream. * * @param ssp system stream partition for which starting offset is requested diff --git a/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java b/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java index 41142c5495..3b81b2d1eb 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java +++ b/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java @@ -35,6 +35,7 @@ import org.apache.samza.Partition; import org.apache.samza.SamzaException; import org.apache.samza.config.Config; +import org.apache.samza.config.StreamConfig; import org.apache.samza.container.TaskName; import org.apache.samza.job.model.TaskMode; import org.apache.samza.storage.kv.Entry; @@ -162,8 +163,8 @@ public StorageEngine getStore(String storeName) { /** * Gets the starting offset for the given side input {@link SystemStreamPartition}. * - * Note: The method doesn't respect {@link org.apache.samza.config.StreamConfigJava#CONSUMER_OFFSET_DEFAULT} and - * {@link org.apache.samza.config.StreamConfigJava#CONSUMER_RESET_OFFSET} configurations. It will use the local offset + * Note: The method doesn't respect {@link StreamConfig#CONSUMER_OFFSET_DEFAULT} and + * {@link StreamConfig#CONSUMER_RESET_OFFSET} configurations. It will use the local offset * file if it is valid, else it will fall back to oldest offset in the stream. * * @param ssp side input system stream partition to get the starting offset for diff --git a/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java b/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java index f22b63a85a..6f00549c32 100644 --- a/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java +++ b/samza-core/src/main/java/org/apache/samza/util/StreamUtil.java @@ -20,7 +20,7 @@ import org.apache.samza.SamzaException; import org.apache.samza.config.Config; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.system.StreamSpec; import org.apache.samza.system.SystemStream; @@ -44,7 +44,7 @@ public static SystemStream getSystemStreamFromNameOrId(Config config, String str String.format("Invalid stream %s. Expected to be of the format streamId or systemName.streamName", stream)); } if (parts.length == 1) { - return new StreamConfigJava(config).streamIdToSystemStream(stream); + return new StreamConfig(config).streamIdToSystemStream(stream); } else { return new SystemStream(parts[0], parts[1]); } @@ -79,11 +79,11 @@ public static String getNameFromSystemStream(SystemStream systemStream) { return systemStream.getSystem() + "." + systemStream.getStream(); } - public static Set getStreamSpecs(Set streamIds, StreamConfigJava streamConfig) { + public static Set getStreamSpecs(Set streamIds, StreamConfig streamConfig) { return streamIds.stream().map(streamId -> getStreamSpec(streamId, streamConfig)).collect(Collectors.toSet()); } - public static StreamSpec getStreamSpec(String streamId, StreamConfigJava streamConfig) { + public static StreamSpec getStreamSpec(String streamId, StreamConfig streamConfig) { String physicalName = streamConfig.getPhysicalName(streamId); String system = streamConfig.getSystem(streamId); Map streamProperties = streamConfig.getStreamProperties(streamId); diff --git a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala index 8a23f892f8..1822c9c659 100644 --- a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala +++ b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala @@ -25,7 +25,7 @@ import java.util.concurrent.ConcurrentHashMap import org.apache.commons.lang3.StringUtils import org.apache.samza.SamzaException import org.apache.samza.annotation.InterfaceStability -import org.apache.samza.config.{Config, StreamConfigJava, SystemConfig} +import org.apache.samza.config.{Config, StreamConfig, SystemConfig} import org.apache.samza.container.TaskName import org.apache.samza.startpoint.{Startpoint, StartpointManager} import org.apache.samza.system.SystemStreamMetadata.OffsetType @@ -79,7 +79,7 @@ object OffsetManager extends Logging { offsetManagerMetrics: OffsetManagerMetrics = new OffsetManagerMetrics) = { debug("Building offset manager for %s." format systemStreamMetadata) - val streamConfig = new StreamConfigJava(config) + val streamConfig = new StreamConfig(config) val offsetSettings = systemStreamMetadata .map { diff --git a/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java b/samza-core/src/main/scala/org/apache/samza/config/StreamConfig.java similarity index 99% rename from samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java rename to samza-core/src/main/scala/org/apache/samza/config/StreamConfig.java index 3cf1e276a9..7477a6ce40 100644 --- a/samza-core/src/main/scala/org/apache/samza/config/StreamConfigJava.java +++ b/samza-core/src/main/scala/org/apache/samza/config/StreamConfig.java @@ -33,7 +33,7 @@ import java.util.Set; import java.util.stream.Collectors; -public class StreamConfigJava extends MapConfig { +public class StreamConfig extends MapConfig { public static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -93,7 +93,7 @@ private Optional nonEmptyOption(String value) { } } - public StreamConfigJava(Config config) { + public StreamConfig(Config config) { super(config); } diff --git a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala index a21dfb6136..b552860225 100644 --- a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala +++ b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala @@ -32,8 +32,7 @@ import java.util.stream.Collectors import com.google.common.annotations.VisibleForTesting import com.google.common.util.concurrent.ThreadFactoryBuilder import org.apache.samza.checkpoint.{CheckpointListener, OffsetManager, OffsetManagerMetrics} -import org.apache.samza.config.StreamConfigJava -import org.apache.samza.config._ +import org.apache.samza.config.{StreamConfig, _} import org.apache.samza.container.disk.DiskSpaceMonitor.Listener import org.apache.samza.container.disk.{DiskQuotaPolicyFactory, DiskSpaceMonitor, NoThrottlingDiskQuotaPolicyFactory, PollingScanDiskSpaceMonitor} import org.apache.samza.container.host.{StatisticsMonitorImpl, SystemMemoryStatistics, SystemStatisticsMonitor} @@ -197,7 +196,7 @@ object SamzaContainer extends Logging { info("Got system names: %s" format systemNames) - val streamConfig = new StreamConfigJava(config) + val streamConfig = new StreamConfig(config) val serdeStreams = systemNames.foldLeft(Set[SystemStream]())(_ ++ streamConfig.getSerdeStreams(_).asScala) info("Got serde streams: %s" format serdeStreams) diff --git a/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala b/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala index 58c1848b66..f66f4c7bb2 100644 --- a/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala +++ b/samza-core/src/main/scala/org/apache/samza/container/TaskInstance.scala @@ -25,7 +25,7 @@ import java.util.concurrent.ScheduledExecutorService import org.apache.samza.SamzaException import org.apache.samza.checkpoint.OffsetManager -import org.apache.samza.config.{Config, StreamConfigJava} +import org.apache.samza.config.{Config, StreamConfig} import org.apache.samza.context._ import org.apache.samza.job.model.{JobModel, TaskModel} import org.apache.samza.scheduler.{CallbackSchedulerImpl, EpochTimeScheduler, ScheduledCallback} @@ -99,7 +99,7 @@ class TaskInstance( private val config: Config = jobContext.getConfig - val streamConfig: StreamConfigJava = new StreamConfigJava(config) + val streamConfig: StreamConfig = new StreamConfig(config) val intermediateStreams: Set[String] = streamConfig.getStreamIds.filter(streamConfig.getIsIntermediateStream).toSet val streamsToDeleteCommittedMessages: Set[String] = streamConfig.getStreamIds.filter(streamConfig.getDeleteCommittedMessages).map(streamConfig.getPhysicalName).toSet diff --git a/samza-core/src/main/scala/org/apache/samza/job/JobRunner.scala b/samza-core/src/main/scala/org/apache/samza/job/JobRunner.scala index 74fb7fbbdd..cbbca1a239 100644 --- a/samza-core/src/main/scala/org/apache/samza/job/JobRunner.scala +++ b/samza-core/src/main/scala/org/apache/samza/job/JobRunner.scala @@ -124,7 +124,7 @@ class JobRunner(config: Config) extends Logging { val diagnosticsSystemStream = StreamUtil.getSystemStreamFromNames(diagnosticsSystemStreamName) val diagnosticsSysAdmin = systemAdmins.getSystemAdmin(diagnosticsSystemStream.getSystem) val diagnosticsStreamSpec = new StreamSpec(DIAGNOSTICS_STREAM_ID, diagnosticsSystemStream.getStream, - diagnosticsSystemStream.getSystem, new StreamConfigJava(config).getStreamProperties(DIAGNOSTICS_STREAM_ID)) + diagnosticsSystemStream.getSystem, new StreamConfig(config).getStreamProperties(DIAGNOSTICS_STREAM_ID)) info("Creating diagnostics stream %s" format diagnosticsSystemStream.getStream) diagnosticsSysAdmin.start() diff --git a/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala b/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala index 7ead386bc5..e3e33d5c80 100644 --- a/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala +++ b/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala @@ -21,8 +21,8 @@ package org.apache.samza.metrics.reporter import org.apache.samza.util.{Logging, StreamUtil, Util} import org.apache.samza.SamzaException -import org.apache.samza.config.{Config, JobConfig, MetricsConfig, SerializerConfig, StreamConfigJava, SystemConfig} -import org.apache.samza.config.StreamConfigJava +import org.apache.samza.config.{Config, JobConfig, MetricsConfig, SerializerConfig, StreamConfig, SystemConfig} +import StreamConfig import org.apache.samza.metrics.MetricsReporter import org.apache.samza.metrics.MetricsReporterFactory import org.apache.samza.metrics.MetricsRegistryMap @@ -63,7 +63,7 @@ class MetricsSnapshotReporterFactory extends MetricsReporterFactory with Logging val producer = systemFactory.getProducer(systemName, config, registry) info("Got producer %s." format producer) - val streamConfig = new StreamConfigJava(config) + val streamConfig = new StreamConfig(config) val streamSerdeName = streamConfig.getStreamMsgSerde(systemStream) val systemSerdeName = systemConfig.getSystemMsgSerde(systemName) diff --git a/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java b/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java index e96cdbe332..9d02393f18 100644 --- a/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java +++ b/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java @@ -47,201 +47,201 @@ public class TestStreamConfig { @Test public void testGetStreamMsgSerde() { String value = "my.msg.serde"; - doTestSamzaProperty(StreamConfigJava.MSG_SERDE, value, + doTestSamzaProperty(StreamConfig.MSG_SERDE, value, (config, systemStream) -> assertEquals(Optional.of(value), config.getStreamMsgSerde(systemStream))); - doTestSamzaProperty(StreamConfigJava.MSG_SERDE, "", + doTestSamzaProperty(StreamConfig.MSG_SERDE, "", (config, systemStream) -> assertEquals(Optional.empty(), config.getStreamMsgSerde(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfigJava.MSG_SERDE, + doTestSamzaPropertyDoesNotExist(StreamConfig.MSG_SERDE, (config, systemStream) -> assertEquals(Optional.empty(), config.getStreamMsgSerde(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfigJava::getStreamMsgSerde); + doTestSamzaPropertyInvalidConfig(StreamConfig::getStreamMsgSerde); } @Test public void testGetStreamKeySerde() { String value = "my.key.serde"; - doTestSamzaProperty(StreamConfigJava.KEY_SERDE, value, + doTestSamzaProperty(StreamConfig.KEY_SERDE, value, (config, systemStream) -> assertEquals(Optional.of(value), config.getStreamKeySerde(systemStream))); - doTestSamzaProperty(StreamConfigJava.KEY_SERDE, "", + doTestSamzaProperty(StreamConfig.KEY_SERDE, "", (config, systemStream) -> assertEquals(Optional.empty(), config.getStreamKeySerde(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfigJava.KEY_SERDE, + doTestSamzaPropertyDoesNotExist(StreamConfig.KEY_SERDE, (config, systemStream) -> assertEquals(Optional.empty(), config.getStreamKeySerde(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfigJava::getStreamKeySerde); + doTestSamzaPropertyInvalidConfig(StreamConfig::getStreamKeySerde); } @Test public void testGetResetOffset() { - doTestSamzaProperty(StreamConfigJava.CONSUMER_RESET_OFFSET, "true", + doTestSamzaProperty(StreamConfig.CONSUMER_RESET_OFFSET, "true", (config, systemStream) -> assertTrue(config.getResetOffset(systemStream))); - doTestSamzaProperty(StreamConfigJava.CONSUMER_RESET_OFFSET, "false", + doTestSamzaProperty(StreamConfig.CONSUMER_RESET_OFFSET, "false", (config, systemStream) -> assertFalse(config.getResetOffset(systemStream))); // if not true/false, then use false - doTestSamzaProperty(StreamConfigJava.CONSUMER_RESET_OFFSET, "unknown_value", + doTestSamzaProperty(StreamConfig.CONSUMER_RESET_OFFSET, "unknown_value", (config, systemStream) -> assertFalse(config.getResetOffset(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfigJava.CONSUMER_RESET_OFFSET, + doTestSamzaPropertyDoesNotExist(StreamConfig.CONSUMER_RESET_OFFSET, (config, systemStream) -> assertFalse(config.getResetOffset(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfigJava::getResetOffset); + doTestSamzaPropertyInvalidConfig(StreamConfig::getResetOffset); } @Test public void testIsResetOffsetConfigured() { - doTestSamzaProperty(StreamConfigJava.CONSUMER_RESET_OFFSET, "true", + doTestSamzaProperty(StreamConfig.CONSUMER_RESET_OFFSET, "true", (config, systemStream) -> assertTrue(config.isResetOffsetConfigured(systemStream))); - doTestSamzaProperty(StreamConfigJava.CONSUMER_RESET_OFFSET, "false", + doTestSamzaProperty(StreamConfig.CONSUMER_RESET_OFFSET, "false", (config, systemStream) -> assertTrue(config.isResetOffsetConfigured(systemStream))); // if not true/false, then use false - doTestSamzaProperty(StreamConfigJava.CONSUMER_RESET_OFFSET, "unknown_value", + doTestSamzaProperty(StreamConfig.CONSUMER_RESET_OFFSET, "unknown_value", (config, systemStream) -> assertTrue(config.isResetOffsetConfigured(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfigJava.CONSUMER_RESET_OFFSET, + doTestSamzaPropertyDoesNotExist(StreamConfig.CONSUMER_RESET_OFFSET, (config, systemStream) -> assertFalse(config.isResetOffsetConfigured(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfigJava::isResetOffsetConfigured); + doTestSamzaPropertyInvalidConfig(StreamConfig::isResetOffsetConfigured); } @Test public void testGetDefaultStreamOffset() { String value = "my_offset_default"; - doTestSamzaProperty(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, value, + doTestSamzaProperty(StreamConfig.CONSUMER_OFFSET_DEFAULT, value, (config, systemStream) -> assertEquals(Optional.of(value), config.getDefaultStreamOffset(systemStream))); - doTestSamzaProperty(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, "", + doTestSamzaProperty(StreamConfig.CONSUMER_OFFSET_DEFAULT, "", (config, systemStream) -> assertEquals(Optional.of(""), config.getDefaultStreamOffset(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, + doTestSamzaPropertyDoesNotExist(StreamConfig.CONSUMER_OFFSET_DEFAULT, (config, systemStream) -> assertEquals(Optional.empty(), - new StreamConfigJava(config).getDefaultStreamOffset(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfigJava::getDefaultStreamOffset); + new StreamConfig(config).getDefaultStreamOffset(systemStream))); + doTestSamzaPropertyInvalidConfig(StreamConfig::getDefaultStreamOffset); } @Test public void testIsDefaultStreamOffsetConfigured() { String value = "my_offset_default"; - doTestSamzaProperty(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, value, + doTestSamzaProperty(StreamConfig.CONSUMER_OFFSET_DEFAULT, value, (config, systemStream) -> assertTrue(config.isDefaultStreamOffsetConfigured(systemStream))); - doTestSamzaProperty(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, "", + doTestSamzaProperty(StreamConfig.CONSUMER_OFFSET_DEFAULT, "", (config, systemStream) -> assertTrue(config.isDefaultStreamOffsetConfigured(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfigJava.CONSUMER_OFFSET_DEFAULT, + doTestSamzaPropertyDoesNotExist(StreamConfig.CONSUMER_OFFSET_DEFAULT, (config, systemStream) -> assertFalse(config.isDefaultStreamOffsetConfigured(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfigJava::isDefaultStreamOffsetConfigured); + doTestSamzaPropertyInvalidConfig(StreamConfig::isDefaultStreamOffsetConfigured); } @Test public void testGetBootstrapEnabled() { - doTestSamzaProperty(StreamConfigJava.BOOTSTRAP, "true", + doTestSamzaProperty(StreamConfig.BOOTSTRAP, "true", (config, systemStream) -> assertTrue(config.getBootstrapEnabled(systemStream))); - doTestSamzaProperty(StreamConfigJava.BOOTSTRAP, "false", + doTestSamzaProperty(StreamConfig.BOOTSTRAP, "false", (config, systemStream) -> assertFalse(config.getBootstrapEnabled(systemStream))); // if not true/false, then use false - doTestSamzaProperty(StreamConfigJava.BOOTSTRAP, "unknown_value", + doTestSamzaProperty(StreamConfig.BOOTSTRAP, "unknown_value", (config, systemStream) -> assertFalse(config.getBootstrapEnabled(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfigJava.BOOTSTRAP, + doTestSamzaPropertyDoesNotExist(StreamConfig.BOOTSTRAP, (config, systemStream) -> assertFalse(config.getBootstrapEnabled(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfigJava::getBootstrapEnabled); + doTestSamzaPropertyInvalidConfig(StreamConfig::getBootstrapEnabled); } @Test public void testGetBroadcastEnabled() { - doTestSamzaProperty(StreamConfigJava.BROADCAST, "true", + doTestSamzaProperty(StreamConfig.BROADCAST, "true", (config, systemStream) -> assertTrue(config.getBroadcastEnabled(systemStream))); - doTestSamzaProperty(StreamConfigJava.BROADCAST, "false", + doTestSamzaProperty(StreamConfig.BROADCAST, "false", (config, systemStream) -> assertFalse(config.getBroadcastEnabled(systemStream))); // if not true/false, then use false - doTestSamzaProperty(StreamConfigJava.BROADCAST, "unknown_value", + doTestSamzaProperty(StreamConfig.BROADCAST, "unknown_value", (config, systemStream) -> assertFalse(config.getBroadcastEnabled(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfigJava.BROADCAST, + doTestSamzaPropertyDoesNotExist(StreamConfig.BROADCAST, (config, systemStream) -> assertFalse(config.getBroadcastEnabled(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfigJava::getBroadcastEnabled); + doTestSamzaPropertyInvalidConfig(StreamConfig::getBroadcastEnabled); } @Test public void testGetPriority() { - doTestSamzaProperty(StreamConfigJava.PRIORITY, "0", + doTestSamzaProperty(StreamConfig.PRIORITY, "0", (config, systemStream) -> assertEquals(0, config.getPriority(systemStream))); - doTestSamzaProperty(StreamConfigJava.PRIORITY, "100", + doTestSamzaProperty(StreamConfig.PRIORITY, "100", (config, systemStream) -> assertEquals(100, config.getPriority(systemStream))); - doTestSamzaProperty(StreamConfigJava.PRIORITY, "-1", + doTestSamzaProperty(StreamConfig.PRIORITY, "-1", (config, systemStream) -> assertEquals(-1, config.getPriority(systemStream))); - doTestSamzaPropertyDoesNotExist(StreamConfigJava.PRIORITY, + doTestSamzaPropertyDoesNotExist(StreamConfig.PRIORITY, (config, systemStream) -> assertEquals(-1, config.getPriority(systemStream))); - doTestSamzaPropertyInvalidConfig(StreamConfigJava::getPriority); + doTestSamzaPropertyInvalidConfig(StreamConfig::getPriority); } @Test public void testGetSerdeStreams() { assertEquals(Collections.emptySet(), - new StreamConfigJava(new MapConfig()).getSerdeStreams(SYSTEM)); + new StreamConfig(new MapConfig()).getSerdeStreams(SYSTEM)); // not key/msg serde property for "streams." - StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); + StreamConfig streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); assertEquals(Collections.emptySet(), streamConfig.getSerdeStreams(SYSTEM)); // not matching system for "streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.KEY_SERDE, UNUSED_VALUE, - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), "otherSystem"))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + StreamConfig.KEY_SERDE, UNUSED_VALUE, + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), "otherSystem"))); assertEquals(Collections.emptySet(), streamConfig.getSerdeStreams(SYSTEM)); // not key/msg serde property for "systems..streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); assertEquals(Collections.emptySet(), streamConfig.getSerdeStreams(SYSTEM)); // not matching system for "systems..streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, "otherSystem", STREAM_ID) + StreamConfigJava.KEY_SERDE, + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, "otherSystem", STREAM_ID) + StreamConfig.KEY_SERDE, UNUSED_VALUE))); assertEquals(Collections.emptySet(), streamConfig.getSerdeStreams(SYSTEM)); // not matching system for "streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.KEY_SERDE, UNUSED_VALUE, - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), "otherSystem"))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + StreamConfig.KEY_SERDE, UNUSED_VALUE, + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), "otherSystem"))); assertEquals(Collections.emptySet(), streamConfig.getSerdeStreams(SYSTEM)); String serdeValue = "my.serde.class"; // key serde for "streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.KEY_SERDE, serdeValue))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + StreamConfig.KEY_SERDE, serdeValue))); assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), streamConfig.getSerdeStreams(SYSTEM)); // msg serde for "streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.MSG_SERDE, serdeValue))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + StreamConfig.MSG_SERDE, serdeValue))); assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), streamConfig.getSerdeStreams(SYSTEM)); // serde for "streams." with physical stream name mapping - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM, - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.KEY_SERDE, serdeValue))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM, + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + StreamConfig.KEY_SERDE, serdeValue))); assertEquals(Collections.singleton(new SystemStream(SYSTEM, PHYSICAL_STREAM)), streamConfig.getSerdeStreams(SYSTEM)); // key serde for "systems..streams." - streamConfig = new StreamConfigJava(new MapConfig( - ImmutableMap.of(String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + StreamConfigJava.KEY_SERDE, + streamConfig = new StreamConfig(new MapConfig( + ImmutableMap.of(String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + StreamConfig.KEY_SERDE, serdeValue))); assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), streamConfig.getSerdeStreams(SYSTEM)); // msg serde for "systems..streams." - streamConfig = new StreamConfigJava(new MapConfig( - ImmutableMap.of(String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + StreamConfigJava.MSG_SERDE, + streamConfig = new StreamConfig(new MapConfig( + ImmutableMap.of(String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + StreamConfig.MSG_SERDE, serdeValue))); assertEquals(Collections.singleton(new SystemStream(SYSTEM, STREAM_ID)), streamConfig.getSerdeStreams(SYSTEM)); // merge several different ways of providing serdes String streamIdWithPhysicalName = "streamIdWithPhysicalName"; - streamConfig = new StreamConfigJava(new MapConfig(new ImmutableMap.Builder() + streamConfig = new StreamConfig(new MapConfig(new ImmutableMap.Builder() // need to map the stream ids to the system .put(JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM) // key and msg serde for "streams." - .put(String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.KEY_SERDE, serdeValue) - .put(String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + StreamConfigJava.MSG_SERDE, serdeValue) + .put(String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + StreamConfig.KEY_SERDE, serdeValue) + .put(String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + StreamConfig.MSG_SERDE, serdeValue) // key serde for "streams." with physical stream name mapping - .put(String.format(StreamConfigJava.STREAM_ID_PREFIX, streamIdWithPhysicalName) + StreamConfigJava.KEY_SERDE, + .put(String.format(StreamConfig.STREAM_ID_PREFIX, streamIdWithPhysicalName) + StreamConfig.KEY_SERDE, serdeValue) - .put(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, streamIdWithPhysicalName), PHYSICAL_STREAM) + .put(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, streamIdWithPhysicalName), PHYSICAL_STREAM) // key serde for "systems..streams." - .put(String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, OTHER_STREAM_ID) + StreamConfigJava.KEY_SERDE, + .put(String.format(StreamConfig.STREAM_PREFIX, SYSTEM, OTHER_STREAM_ID) + StreamConfig.KEY_SERDE, serdeValue) .build())); assertEquals(Sets.newHashSet(new SystemStream(SYSTEM, STREAM_ID), new SystemStream(SYSTEM, PHYSICAL_STREAM), @@ -250,44 +250,44 @@ public void testGetSerdeStreams() { @Test public void testGetStreamProperties() { - assertEquals(new MapConfig(), new StreamConfigJava(new MapConfig()).getStreamProperties(STREAM_ID)); + assertEquals(new MapConfig(), new StreamConfig(new MapConfig()).getStreamProperties(STREAM_ID)); String propertyName = "stream.property.name"; // BEGIN: tests in which properties cannot be found in the config // not matching stream id for "streams." - StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, OTHER_STREAM_ID) + propertyName, UNUSED_VALUE))); + StreamConfig streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, OTHER_STREAM_ID) + propertyName, UNUSED_VALUE))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // not matching stream id for "systems..streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, OTHER_STREAM_ID) + propertyName, UNUSED_VALUE, - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, OTHER_STREAM_ID), SYSTEM))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, OTHER_STREAM_ID) + propertyName, UNUSED_VALUE, + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, OTHER_STREAM_ID), SYSTEM))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // no system mapping when using "systems..streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // ignore property with "samza" prefix for "streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // ignore property with "samza" prefix for "streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE, - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE, + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(STREAM_ID)); // should not map physical name back to stream id if physical name is passed as stream id - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); assertEquals(new MapConfig(), streamConfig.getStreamProperties(PHYSICAL_STREAM)); // BEGIN: tests in which properties can be found in the config @@ -295,65 +295,65 @@ public void testGetStreamProperties() { String propertyValue = "value"; // "streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + propertyName, propertyValue))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + propertyName, propertyValue))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // "systems..streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, propertyValue, - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, propertyValue, + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // "systems..default.stream." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, propertyValue, - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // use physical name mapping for "systems..streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, PHYSICAL_STREAM) + propertyName, propertyValue, + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, PHYSICAL_STREAM) + propertyName, propertyValue, // should not use stream id since there is physical stream - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // "streams." should override "systems..streams." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + propertyName, propertyValue, + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + propertyName, propertyValue, // should not use "systems..streams." since there is a "streams." config - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // "systems..streams." should override "systems..default.stream." - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, propertyValue, + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, propertyValue, // should not use "systems..default.stream." since there is a "systems..streams." String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, UNUSED_VALUE, - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of(propertyName, propertyValue)), streamConfig.getStreamProperties(STREAM_ID)); // merge multiple ways of specifying configs - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( // "streams." - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + "from.stream.id.property", "fromStreamIdValue", + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + "from.stream.id.property", "fromStreamIdValue", // second "streams." property - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + "from.stream.id.other.property", + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + "from.stream.id.other.property", "fromStreamIdOtherValue", // "systems..streams." - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + "from.system.stream.property", + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + "from.system.stream.property", "fromSystemStreamValue", // need to map the stream id to a system - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))); assertEquals(new MapConfig(ImmutableMap.of( "from.stream.id.property", "fromStreamIdValue", "from.stream.id.other.property", "fromStreamIdOtherValue", @@ -363,131 +363,131 @@ public void testGetStreamProperties() { @Test public void testGetSystem() { - assertNull(new StreamConfigJava(new MapConfig()).getSystem(STREAM_ID)); + assertNull(new StreamConfig(new MapConfig()).getSystem(STREAM_ID)); // system is specified directly - StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + StreamConfig streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, JobConfig.JOB_DEFAULT_SYSTEM, "otherSystem", - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, OTHER_STREAM_ID), "otherSystem"))); + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, OTHER_STREAM_ID), "otherSystem"))); assertEquals(SYSTEM, streamConfig.getSystem(STREAM_ID)); // fall back to job default system - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, OTHER_STREAM_ID), "otherSystem"))); + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, OTHER_STREAM_ID), "otherSystem"))); assertEquals(SYSTEM, streamConfig.getSystem(STREAM_ID)); } @Test public void testGetPhysicalName() { - assertEquals(STREAM_ID, new StreamConfigJava(new MapConfig()).getPhysicalName(STREAM_ID)); + assertEquals(STREAM_ID, new StreamConfig(new MapConfig()).getPhysicalName(STREAM_ID)); // ignore mapping for other stream ids - StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig( - ImmutableMap.of(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, OTHER_STREAM_ID), PHYSICAL_STREAM))); + StreamConfig streamConfig = new StreamConfig(new MapConfig( + ImmutableMap.of(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, OTHER_STREAM_ID), PHYSICAL_STREAM))); assertEquals(STREAM_ID, streamConfig.getPhysicalName(STREAM_ID)); - streamConfig = new StreamConfigJava(new MapConfig( - ImmutableMap.of(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfig(new MapConfig( + ImmutableMap.of(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); assertEquals(PHYSICAL_STREAM, streamConfig.getPhysicalName(STREAM_ID)); } @Test public void testGetIsIntermediateStream() { - assertFalse(new StreamConfigJava(new MapConfig()).getIsIntermediateStream(STREAM_ID)); + assertFalse(new StreamConfig(new MapConfig()).getIsIntermediateStream(STREAM_ID)); // ignore mapping for other stream ids - StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig( - ImmutableMap.of(String.format(StreamConfigJava.IS_INTERMEDIATE_FOR_STREAM_ID, OTHER_STREAM_ID), "true"))); + StreamConfig streamConfig = new StreamConfig(new MapConfig( + ImmutableMap.of(String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID, OTHER_STREAM_ID), "true"))); assertFalse(streamConfig.getIsIntermediateStream(STREAM_ID)); // do not use stream id property if physical name is passed as input - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.IS_INTERMEDIATE_FOR_STREAM_ID, STREAM_ID), "true", - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID, STREAM_ID), "true", + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); assertFalse(streamConfig.getIsIntermediateStream(PHYSICAL_STREAM)); - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.IS_INTERMEDIATE_FOR_STREAM_ID, STREAM_ID), "true"))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID, STREAM_ID), "true"))); assertTrue(streamConfig.getIsIntermediateStream(STREAM_ID)); - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.IS_INTERMEDIATE_FOR_STREAM_ID, STREAM_ID), "false"))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID, STREAM_ID), "false"))); assertFalse(streamConfig.getIsIntermediateStream(STREAM_ID)); } @Test public void testGetDeleteCommittedMessages() { - assertFalse(new StreamConfigJava(new MapConfig()).getDeleteCommittedMessages(STREAM_ID)); + assertFalse(new StreamConfig(new MapConfig()).getDeleteCommittedMessages(STREAM_ID)); // ignore mapping for other stream ids - StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig( - ImmutableMap.of(String.format(StreamConfigJava.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, OTHER_STREAM_ID), + StreamConfig streamConfig = new StreamConfig(new MapConfig( + ImmutableMap.of(String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, OTHER_STREAM_ID), "true"))); assertFalse(streamConfig.getDeleteCommittedMessages(STREAM_ID)); // do not use stream id property if physical name is passed as input - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, STREAM_ID), "true", - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, STREAM_ID), "true", + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); assertFalse(streamConfig.getDeleteCommittedMessages(PHYSICAL_STREAM)); - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, STREAM_ID), "true"))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, STREAM_ID), "true"))); assertTrue(streamConfig.getDeleteCommittedMessages(STREAM_ID)); - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, STREAM_ID), "false"))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID, STREAM_ID), "false"))); assertFalse(streamConfig.getDeleteCommittedMessages(STREAM_ID)); } @Test public void testGetIsBounded() { - assertFalse(new StreamConfigJava(new MapConfig()).getIsBounded(STREAM_ID)); + assertFalse(new StreamConfig(new MapConfig()).getIsBounded(STREAM_ID)); // ignore mapping for other stream ids - StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig( - ImmutableMap.of(String.format(StreamConfigJava.IS_BOUNDED_FOR_STREAM_ID, OTHER_STREAM_ID), + StreamConfig streamConfig = new StreamConfig(new MapConfig( + ImmutableMap.of(String.format(StreamConfig.IS_BOUNDED_FOR_STREAM_ID, OTHER_STREAM_ID), "true"))); assertFalse(streamConfig.getIsBounded(STREAM_ID)); // do not use stream id property if physical name is passed as input - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.IS_BOUNDED_FOR_STREAM_ID, STREAM_ID), "true", - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.IS_BOUNDED_FOR_STREAM_ID, STREAM_ID), "true", + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))); assertFalse(streamConfig.getIsBounded(PHYSICAL_STREAM)); - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.IS_BOUNDED_FOR_STREAM_ID, STREAM_ID), "true"))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.IS_BOUNDED_FOR_STREAM_ID, STREAM_ID), "true"))); assertTrue(streamConfig.getIsBounded(STREAM_ID)); - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.IS_BOUNDED_FOR_STREAM_ID, STREAM_ID), "false"))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.IS_BOUNDED_FOR_STREAM_ID, STREAM_ID), "false"))); assertFalse(streamConfig.getIsBounded(STREAM_ID)); } @Test public void testGetStreamIds() { assertEquals(ImmutableList.of(), ImmutableList.copyOf( - new StreamConfigJava(new MapConfig()).getStreamIds())); + new StreamConfig(new MapConfig()).getStreamIds())); - StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + ".property", "value"))); + StreamConfig streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + ".property", "value"))); assertEquals(ImmutableList.of(STREAM_ID), ImmutableList.copyOf(streamConfig.getStreamIds())); - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + ".property.subProperty", "value"))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + ".property.subProperty", "value"))); assertEquals(ImmutableList.of(STREAM_ID), ImmutableList.copyOf(streamConfig.getStreamIds())); - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + ".property0", "value", - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + ".property1", "value", - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + ".property.subProperty0", "value", - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + ".property.subProperty1", "value", - String.format(StreamConfigJava.STREAM_ID_PREFIX, OTHER_STREAM_ID) + ".property", "value"))); + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + ".property0", "value", + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + ".property1", "value", + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + ".property.subProperty0", "value", + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + ".property.subProperty1", "value", + String.format(StreamConfig.STREAM_ID_PREFIX, OTHER_STREAM_ID) + ".property", "value"))); assertEquals(ImmutableList.of(STREAM_ID, OTHER_STREAM_ID), ImmutableList.copyOf(streamConfig.getStreamIds())); } @@ -506,63 +506,63 @@ private static void doTestSamzaProperty(String propertyName, String propertyValu */ private static void doTestSamzaPropertyAccess(String propertyName, String value, SamzaPropertyAssertion assertion) { // streams.. - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + propertyName, value, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM))), SYSTEM_STREAM); // systems..streams.. where stream id has no specified physical stream - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, // specify the system for the stream id - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))), + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))), SYSTEM_STREAM); // systems..streams.. where stream is the streamId, system is from job default system - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, // use job default system to get the system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM))), SYSTEM_STREAM); // systems..default.stream. where stream id has no specified physical stream - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // specify the system for the stream id - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))), + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM))), SYSTEM_STREAM); // systems..streams.. where no system mapping (fall back to SystemStream) - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, // map the stream id to the physical stream - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); // systems..default.stream. where no system mapping (fall back to SystemStream) - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // map the stream id to the physical stream - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); // systems..streams.. where stream id has a physical name but property is from stream id - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, // use job default system to get the system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); // systems..default.stream. where stream id has a physical name but property is from stream id - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // use job default system to get the system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); } @@ -572,39 +572,39 @@ private static void doTestSamzaPropertyAccess(String propertyName, String value, private static void doTestSamzaPropertyAccessWithPhysicalStream(String propertyName, String value, SamzaPropertyAssertion assertion) { // streams.. - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + propertyName, value, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); // systems..streams.. with a specific system for the stream id - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, PHYSICAL_STREAM) + propertyName, value, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, PHYSICAL_STREAM) + propertyName, value, // specify the system for the stream id - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, // map the stream id to the physical stream - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); // systems..streams.. with system coming from job default system - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, PHYSICAL_STREAM) + propertyName, value, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, PHYSICAL_STREAM) + propertyName, value, // use job default system to get the system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); // systems..default.stream. - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // specify the system for the stream id - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, // map the stream id to the physical stream - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); } @@ -615,19 +615,19 @@ private static void doTestSamzaPropertyAccessWithPhysicalStream(String propertyN */ private static void doTestSamzaPropertyPriority(String propertyName, String value, SamzaPropertyAssertion assertion) { // streams.. vs. systems..streams.. - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + propertyName, value, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // this config should not be used - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE))), + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE))), SYSTEM_STREAM); // systems..streams.. vs. systems..default.stream. - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, // specify the system for the stream id - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, // this config should not be used String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, UNUSED_VALUE))), SYSTEM_STREAM); @@ -642,8 +642,8 @@ private static void doTestSamzaPropertyPriority(String propertyName, String valu * systems..streams.. without a system mapping in the config vs. * systems..default.stream. without a system mapping in the config */ - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, value, // this config should not be used String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, UNUSED_VALUE))), SYSTEM_STREAM); @@ -655,19 +655,19 @@ private static void doTestSamzaPropertyPriority(String propertyName, String valu */ private static void doTestSamzaPropertyMultipleStreams(String propertyName, String value, SamzaPropertyAssertion assertion) { // systems..default.stream. where stream id has no specified physical stream - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( String.format(SystemConfig.SYSTEM_DEFAULT_STREAMS_PREFIX_FORMAT, SYSTEM) + propertyName, value, // specify the systems for the streams - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, - String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, OTHER_STREAM_ID), SYSTEM))), + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID), SYSTEM, + String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, OTHER_STREAM_ID), SYSTEM))), SYSTEM_STREAM); } private static void doTestSamzaPropertyInvalidConfig(SamzaPropertyLookup lookup) { // configure physical stream and have mapping from stream id to physical stream - StreamConfigJava streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM, - String.format(StreamConfigJava.STREAM_ID_PREFIX, PHYSICAL_STREAM) + ".property", "value", + StreamConfig streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM, + String.format(StreamConfig.STREAM_ID_PREFIX, PHYSICAL_STREAM) + ".property", "value", JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM))); try { lookup.doLookup(streamConfig, SYSTEM_STREAM_PHYSICAL); @@ -677,9 +677,9 @@ private static void doTestSamzaPropertyInvalidConfig(SamzaPropertyLookup lookup) } // two separate stream ids map to same physical stream - streamConfig = new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM, - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, OTHER_STREAM_ID), PHYSICAL_STREAM, + streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM, + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, OTHER_STREAM_ID), PHYSICAL_STREAM, JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM))); try { lookup.doLookup(streamConfig, SYSTEM_STREAM_PHYSICAL); @@ -690,42 +690,42 @@ private static void doTestSamzaPropertyInvalidConfig(SamzaPropertyLookup lookup) } private static void doTestSamzaPropertyDoesNotExist(String propertyName, SamzaPropertyAssertion assertion) { - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( // just put in some value which will be ignored - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))), + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + SAMZA_IGNORED_PROPERTY, UNUSED_VALUE))), SYSTEM_STREAM); /* * Won't use streams.. if streamId is mapped to a physical stream */ - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_ID_PREFIX, STREAM_ID) + propertyName, UNUSED_VALUE, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + propertyName, UNUSED_VALUE, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); /* * Won't use systems..streams.. if stream is mapped to a physical stream and the property * is only specified using the physical stream */ - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, PHYSICAL_STREAM) + propertyName, UNUSED_VALUE, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, PHYSICAL_STREAM) + propertyName, UNUSED_VALUE, // all streams need to have a system JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM, // map the stream id to the physical stream - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM); /* * Won't use systems..streams.. if stream is mapped to a physical stream and there is no * system mapping for the stream id */ - assertion.doAssertion(new StreamConfigJava(new MapConfig(ImmutableMap.of( - String.format(StreamConfigJava.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, + assertion.doAssertion(new StreamConfig(new MapConfig(ImmutableMap.of( + String.format(StreamConfig.STREAM_PREFIX, SYSTEM, STREAM_ID) + propertyName, UNUSED_VALUE, // map the stream id to the physical stream - String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), + String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID), PHYSICAL_STREAM))), SYSTEM_STREAM_PHYSICAL); } @@ -734,7 +734,7 @@ private static void doTestSamzaPropertyDoesNotExist(String propertyName, SamzaPr */ @FunctionalInterface private interface SamzaPropertyAssertion { - void doAssertion(StreamConfigJava streamConfig, SystemStream systemStream); + void doAssertion(StreamConfig streamConfig, SystemStream systemStream); } /** @@ -742,6 +742,6 @@ private interface SamzaPropertyAssertion { */ @FunctionalInterface private interface SamzaPropertyLookup { - void doLookup(StreamConfigJava streamConfig, SystemStream systemStream); + void doLookup(StreamConfig streamConfig, SystemStream systemStream); } } \ No newline at end of file diff --git a/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java b/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java index 36ecf58f5c..f8888f6ae3 100644 --- a/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java +++ b/samza-core/src/test/java/org/apache/samza/config/TestSystemConfig.java @@ -158,13 +158,13 @@ public void testGetSystemKeySerde() { // value specified explicitly Config config = - new MapConfig(ImmutableMap.of(defaultStreamPrefixSystem1 + StreamConfigJava.KEY_SERDE, system1KeySerde)); + new MapConfig(ImmutableMap.of(defaultStreamPrefixSystem1 + StreamConfig.KEY_SERDE, system1KeySerde)); SystemConfig systemConfig = new SystemConfig(config); assertEquals(system1KeySerde, systemConfig.getSystemKeySerde(MOCK_SYSTEM_NAME1).get()); // default stream property is unspecified, try fall back config key config = new MapConfig( - ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfigJava.KEY_SERDE, + ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig.KEY_SERDE, system1KeySerde)); systemConfig = new SystemConfig(config); assertEquals(system1KeySerde, systemConfig.getSystemKeySerde(MOCK_SYSTEM_NAME1).get()); @@ -172,15 +172,15 @@ public void testGetSystemKeySerde() { // default stream property is empty string, try fall back config key config = new MapConfig(ImmutableMap.of( // default stream property is empty - defaultStreamPrefixSystem1 + StreamConfigJava.KEY_SERDE, "", + defaultStreamPrefixSystem1 + StreamConfig.KEY_SERDE, "", // fall back entry - String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfigJava.KEY_SERDE, system1KeySerde)); + String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig.KEY_SERDE, system1KeySerde)); systemConfig = new SystemConfig(config); assertEquals(system1KeySerde, systemConfig.getSystemKeySerde(MOCK_SYSTEM_NAME1).get()); // default stream property is unspecified, fall back is also empty config = new MapConfig( - ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfigJava.KEY_SERDE, + ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig.KEY_SERDE, "")); systemConfig = new SystemConfig(config); assertFalse(systemConfig.getSystemKeySerde(MOCK_SYSTEM_NAME1).isPresent()); @@ -198,13 +198,13 @@ public void testGetSystemMsgSerde() { // value specified explicitly Config config = - new MapConfig(ImmutableMap.of(defaultStreamPrefixSystem1 + StreamConfigJava.MSG_SERDE, system1MsgSerde)); + new MapConfig(ImmutableMap.of(defaultStreamPrefixSystem1 + StreamConfig.MSG_SERDE, system1MsgSerde)); SystemConfig systemConfig = new SystemConfig(config); assertEquals(system1MsgSerde, systemConfig.getSystemMsgSerde(MOCK_SYSTEM_NAME1).get()); // default stream property is unspecified, try fall back config msg config = new MapConfig( - ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfigJava.MSG_SERDE, + ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig.MSG_SERDE, system1MsgSerde)); systemConfig = new SystemConfig(config); assertEquals(system1MsgSerde, systemConfig.getSystemMsgSerde(MOCK_SYSTEM_NAME1).get()); @@ -212,15 +212,15 @@ public void testGetSystemMsgSerde() { // default stream property is empty string, try fall back config msg config = new MapConfig(ImmutableMap.of( // default stream property is empty - defaultStreamPrefixSystem1 + StreamConfigJava.MSG_SERDE, "", + defaultStreamPrefixSystem1 + StreamConfig.MSG_SERDE, "", // fall back entry - String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfigJava.MSG_SERDE, system1MsgSerde)); + String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig.MSG_SERDE, system1MsgSerde)); systemConfig = new SystemConfig(config); assertEquals(system1MsgSerde, systemConfig.getSystemMsgSerde(MOCK_SYSTEM_NAME1).get()); // default stream property is unspecified, fall back is also empty config = new MapConfig( - ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfigJava.MSG_SERDE, + ImmutableMap.of(String.format(SystemConfig.SYSTEM_ID_PREFIX, MOCK_SYSTEM_NAME1) + StreamConfig.MSG_SERDE, "")); systemConfig = new SystemConfig(config); assertFalse(systemConfig.getSystemMsgSerde(MOCK_SYSTEM_NAME1).isPresent()); diff --git a/samza-core/src/test/java/org/apache/samza/execution/TestExecutionPlanner.java b/samza-core/src/test/java/org/apache/samza/execution/TestExecutionPlanner.java index 7c721839d7..63d290a17a 100644 --- a/samza-core/src/test/java/org/apache/samza/execution/TestExecutionPlanner.java +++ b/samza-core/src/test/java/org/apache/samza/execution/TestExecutionPlanner.java @@ -40,7 +40,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.config.TaskConfig; import org.apache.samza.serializers.StringSerde; import org.apache.samza.system.descriptors.GenericInputDescriptor; @@ -640,7 +640,7 @@ public void testDefaultPartitions() { @Test public void testBroadcastConfig() { Map map = new HashMap<>(config); - map.put(String.format(StreamConfigJava.BROADCAST_FOR_STREAM_ID, "input1"), "true"); + map.put(String.format(StreamConfig.BROADCAST_FOR_STREAM_ID, "input1"), "true"); Config cfg = new MapConfig(map); ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager); diff --git a/samza-core/src/test/java/org/apache/samza/execution/TestStreamEdge.java b/samza-core/src/test/java/org/apache/samza/execution/TestStreamEdge.java index 366e017207..94275dd660 100644 --- a/samza-core/src/test/java/org/apache/samza/execution/TestStreamEdge.java +++ b/samza-core/src/test/java/org/apache/samza/execution/TestStreamEdge.java @@ -22,7 +22,7 @@ import org.apache.samza.config.ApplicationConfig; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.system.StreamSpec; import org.junit.Test; @@ -67,7 +67,7 @@ public void testGenerateConfig() { StreamSpec spec = new StreamSpec("stream-1", "physical-stream-1", "system-1", Collections.singletonMap("property1", "haha")); StreamEdge edge = new StreamEdge(spec, false, false, new MapConfig()); Config config = edge.generateConfig(); - StreamConfigJava streamConfig = new StreamConfigJava(config); + StreamConfig streamConfig = new StreamConfig(config); assertEquals(streamConfig.getSystem(spec.getId()), "system-1"); assertEquals(streamConfig.getPhysicalName(spec.getId()), "physical-stream-1"); assertEquals(streamConfig.getIsIntermediateStream(spec.getId()), false); @@ -77,12 +77,12 @@ public void testGenerateConfig() { spec = new StreamSpec("stream-1", "physical-stream-1", "system-1", Collections.singletonMap("property1", "haha")); edge = new StreamEdge(spec, false, false, new MapConfig()); config = edge.generateConfig(); - streamConfig = new StreamConfigJava(config); + streamConfig = new StreamConfig(config); // intermediate stream edge = new StreamEdge(spec, true, false, new MapConfig()); config = edge.generateConfig(); - streamConfig = new StreamConfigJava(config); + streamConfig = new StreamConfig(config); assertEquals(streamConfig.getIsIntermediateStream(spec.getId()), true); assertEquals(streamConfig.getPriority(spec.toSystemStream()), Integer.MAX_VALUE); } diff --git a/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImplGraph.java b/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImplGraph.java index 99ec93b9f9..579c028831 100644 --- a/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImplGraph.java +++ b/samza-core/src/test/java/org/apache/samza/operators/impl/TestOperatorImplGraph.java @@ -26,7 +26,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.container.TaskName; import org.apache.samza.context.Context; import org.apache.samza.context.MockContext; @@ -504,7 +504,7 @@ public void testGetOutputToInputStreams() { }, config); Multimap outputToInput = - OperatorImplGraph.getIntermediateToInputStreamsMap(graphSpec.getOperatorSpecGraph(), new StreamConfigJava(config)); + OperatorImplGraph.getIntermediateToInputStreamsMap(graphSpec.getOperatorSpecGraph(), new StreamConfig(config)); Collection inputs = outputToInput.get(new SystemStream(intSystem, intStreamId2)); assertEquals(inputs.size(), 2); assertTrue(inputs.contains(new SystemStream(inputSystem, inputStreamId1))); diff --git a/samza-core/src/test/java/org/apache/samza/testUtils/StreamTestUtils.java b/samza-core/src/test/java/org/apache/samza/testUtils/StreamTestUtils.java index 976ccf57b1..6ad245c383 100644 --- a/samza-core/src/test/java/org/apache/samza/testUtils/StreamTestUtils.java +++ b/samza-core/src/test/java/org/apache/samza/testUtils/StreamTestUtils.java @@ -19,7 +19,7 @@ package org.apache.samza.testUtils; import java.util.Map; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; public class StreamTestUtils { @@ -33,7 +33,7 @@ public class StreamTestUtils { */ public static void addStreamConfigs(Map configs, String streamId, String systemName, String physicalName) { - configs.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, streamId), systemName); - configs.put(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, streamId), physicalName); + configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, streamId), systemName); + configs.put(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, streamId), physicalName); } } \ No newline at end of file diff --git a/samza-core/src/test/java/org/apache/samza/util/TestStreamUtil.java b/samza-core/src/test/java/org/apache/samza/util/TestStreamUtil.java index 9573ad1d89..d7db63ab33 100644 --- a/samza-core/src/test/java/org/apache/samza/util/TestStreamUtil.java +++ b/samza-core/src/test/java/org/apache/samza/util/TestStreamUtil.java @@ -21,7 +21,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.JobConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.system.StreamSpec; import org.junit.Test; @@ -50,10 +50,10 @@ public class TestStreamUtil { @Test public void testGetStreamWithPhysicalNameInConfig() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, - StreamConfigJava.SYSTEM, TEST_SYSTEM); + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfig.SYSTEM, TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(TEST_PHYSICAL_NAME, spec.getPhysicalName()); } @@ -63,9 +63,9 @@ public void testGetStreamWithPhysicalNameInConfig() { @Test public void testGetStreamWithoutPhysicalNameInConfig() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.SYSTEM, TEST_SYSTEM); + StreamConfig.SYSTEM, TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(STREAM_ID, spec.getPhysicalName()); } @@ -74,10 +74,10 @@ public void testGetStreamWithoutPhysicalNameInConfig() { @Test public void testGetStreamWithSystemAtStreamScopeInConfig() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, - StreamConfigJava.SYSTEM, TEST_SYSTEM); + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfig.SYSTEM, TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(TEST_SYSTEM, spec.getSystemName()); } @@ -86,10 +86,10 @@ public void testGetStreamWithSystemAtStreamScopeInConfig() { @Test public void testGetStreamWithSystemAtDefaultScopeInConfig() { Config config = addConfigs(buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME), + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME), JobConfig.JOB_DEFAULT_SYSTEM, TEST_DEFAULT_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(TEST_DEFAULT_SYSTEM, spec.getSystemName()); } @@ -98,11 +98,11 @@ public void testGetStreamWithSystemAtDefaultScopeInConfig() { @Test public void testGetStreamWithSystemAtBothScopesInConfig() { Config config = addConfigs(buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, - StreamConfigJava.SYSTEM, TEST_SYSTEM), + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfig.SYSTEM, TEST_SYSTEM), JobConfig.JOB_DEFAULT_SYSTEM, TEST_DEFAULT_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(TEST_SYSTEM, spec.getSystemName()); } @@ -111,9 +111,9 @@ public void testGetStreamWithSystemAtBothScopesInConfig() { @Test(expected = IllegalArgumentException.class) public void testGetStreamWithOutSystemInConfig() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME); + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(TEST_SYSTEM, spec.getSystemName()); } @@ -122,13 +122,13 @@ public void testGetStreamWithOutSystemInConfig() { @Test public void testGetStreamPropertiesPassthrough() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, - StreamConfigJava.SYSTEM, TEST_SYSTEM, + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfig.SYSTEM, TEST_SYSTEM, "systemProperty1", "systemValue1", "systemProperty2", "systemValue2", "systemProperty3", "systemValue3"); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); Map properties = spec.getConfig(); assertEquals(3, properties.size()); @@ -144,35 +144,35 @@ public void testGetStreamPropertiesPassthrough() { @Test public void testGetStreamSamzaPropertiesOmitted() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, - StreamConfigJava.SYSTEM, TEST_SYSTEM, + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfig.SYSTEM, TEST_SYSTEM, "systemProperty1", "systemValue1", "systemProperty2", "systemValue2", "systemProperty3", "systemValue3"); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); Map properties = spec.getConfig(); assertEquals(3, properties.size()); - assertNull(properties.get(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID))); - assertNull(properties.get(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID))); - assertNull(spec.get(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID))); - assertNull(spec.get(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, STREAM_ID))); + assertNull(properties.get(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID))); + assertNull(properties.get(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID))); + assertNull(spec.get(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, STREAM_ID))); + assertNull(spec.get(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, STREAM_ID))); } @Test public void testStreamConfigOverrides() { final String sysStreamPrefix = String.format("systems.%s.streams.%s.", TEST_SYSTEM, TEST_PHYSICAL_NAME); Config config = addConfigs(buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, - StreamConfigJava.SYSTEM, TEST_SYSTEM, + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfig.SYSTEM, TEST_SYSTEM, "systemProperty1", "systemValue1", "systemProperty2", "systemValue2", "systemProperty3", "systemValue3"), sysStreamPrefix + "systemProperty4", "systemValue4", sysStreamPrefix + "systemProperty2", "systemValue8"); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); Map properties = spec.getConfig(); assertEquals(4, properties.size()); @@ -184,14 +184,14 @@ public void testStreamConfigOverrides() { @Test public void testStreamConfigOverridesWithSystemDefaults() { Config config = addConfigs(buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, - StreamConfigJava.SYSTEM, TEST_SYSTEM, + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfig.SYSTEM, TEST_SYSTEM, "segment.bytes", "5309"), String.format("systems.%s.default.stream.replication.factor", TEST_SYSTEM), "4", // System default property String.format("systems.%s.default.stream.segment.bytest", TEST_SYSTEM), "867" ); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); Map properties = spec.getConfig(); assertEquals(3, properties.size()); @@ -203,10 +203,10 @@ public void testStreamConfigOverridesWithSystemDefaults() { @Test public void testGetStreamPhysicalNameArgSimple() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME2, // This should be ignored because of the explicit arg - StreamConfigJava.SYSTEM, TEST_SYSTEM); + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME2, // This should be ignored because of the explicit arg + StreamConfig.SYSTEM, TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(STREAM_ID, spec.getId()); assertEquals(TEST_PHYSICAL_NAME2, spec.getPhysicalName()); @@ -217,10 +217,10 @@ public void testGetStreamPhysicalNameArgSimple() { @Test public void testGetStreamPhysicalNameArgSpecialCharacters() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME_SPECIAL_CHARS, - StreamConfigJava.SYSTEM, TEST_SYSTEM); + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME_SPECIAL_CHARS, + StreamConfig.SYSTEM, TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(TEST_PHYSICAL_NAME_SPECIAL_CHARS, spec.getPhysicalName()); } @@ -228,10 +228,10 @@ public void testGetStreamPhysicalNameArgSpecialCharacters() { @Test public void testGetStreamPhysicalNameArgNull() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, null, - StreamConfigJava.SYSTEM, TEST_SYSTEM); + StreamConfig.PHYSICAL_NAME, null, + StreamConfig.SYSTEM, TEST_SYSTEM); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertNull(spec.getPhysicalName()); } @@ -239,10 +239,10 @@ public void testGetStreamPhysicalNameArgNull() { @Test public void testGetStreamSystemNameArgValid() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, // This should be ignored because of the explicit arg - StreamConfigJava.SYSTEM, TEST_SYSTEM); // This too + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, // This should be ignored because of the explicit arg + StreamConfig.SYSTEM, TEST_SYSTEM); // This too - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(STREAM_ID, spec.getId()); assertEquals(TEST_PHYSICAL_NAME, spec.getPhysicalName()); @@ -253,57 +253,57 @@ public void testGetStreamSystemNameArgValid() { @Test(expected = IllegalArgumentException.class) public void testGetStreamSystemNameArgInvalid() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, - StreamConfigJava.SYSTEM, TEST_SYSTEM_INVALID); + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfig.SYSTEM, TEST_SYSTEM_INVALID); - StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); } // Empty strings are NOT allowed for system name, because it's used as an identifier in the config. @Test(expected = IllegalArgumentException.class) public void testGetStreamSystemNameArgEmpty() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, - StreamConfigJava.SYSTEM, ""); + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfig.SYSTEM, ""); - StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); } // Null is not allowed IllegalArgumentException system name. @Test(expected = IllegalArgumentException.class) public void testGetStreamSystemNameArgNull() { Config config = buildStreamConfig(STREAM_ID, - StreamConfigJava.PHYSICAL_NAME, TEST_PHYSICAL_NAME, - StreamConfigJava.SYSTEM, null); + StreamConfig.PHYSICAL_NAME, TEST_PHYSICAL_NAME, + StreamConfig.SYSTEM, null); - StreamUtil.getStreamSpec(STREAM_ID, new StreamConfigJava(config)); + StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); } // Special characters are NOT allowed for streamId, because it's used as an identifier in the config. @Test(expected = IllegalArgumentException.class) public void testGetStreamStreamIdInvalid() { Config config = buildStreamConfig(STREAM_ID_INVALID, - StreamConfigJava.SYSTEM, TEST_SYSTEM); + StreamConfig.SYSTEM, TEST_SYSTEM); - StreamUtil.getStreamSpec(STREAM_ID_INVALID, new StreamConfigJava(config)); + StreamUtil.getStreamSpec(STREAM_ID_INVALID, new StreamConfig(config)); } // Empty strings are NOT allowed for streamId, because it's used as an identifier in the config. @Test(expected = IllegalArgumentException.class) public void testGetStreamStreamIdEmpty() { Config config = buildStreamConfig("", - StreamConfigJava.SYSTEM, TEST_SYSTEM); + StreamConfig.SYSTEM, TEST_SYSTEM); - StreamUtil.getStreamSpec("", new StreamConfigJava(config)); + StreamUtil.getStreamSpec("", new StreamConfig(config)); } // Null is not allowed for streamId. @Test(expected = IllegalArgumentException.class) public void testGetStreamStreamIdNull() { Config config = buildStreamConfig(null, - StreamConfigJava.SYSTEM, TEST_SYSTEM); + StreamConfig.SYSTEM, TEST_SYSTEM); - StreamUtil.getStreamSpec(null, new StreamConfigJava(config)); + StreamUtil.getStreamSpec(null, new StreamConfig(config)); } @@ -312,7 +312,7 @@ public void testGetStreamStreamIdNull() { private Config buildStreamConfig(String streamId, String... kvs) { // inject streams.x. into each key for (int i = 0; i < kvs.length - 1; i += 2) { - kvs[i] = String.format(StreamConfigJava.STREAM_ID_PREFIX, streamId) + kvs[i]; + kvs[i] = String.format(StreamConfig.STREAM_ID_PREFIX, streamId) + kvs[i]; } return buildConfig(kvs); } diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index 78c26f360a..b6c9ba1e86 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -45,7 +45,7 @@ import org.apache.samza.config.Config; import org.apache.samza.config.KafkaConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.config.SystemConfig; import org.apache.samza.startpoint.Startpoint; import org.apache.samza.startpoint.StartpointOldest; @@ -66,7 +66,6 @@ import scala.Function0; import scala.Function1; import scala.Function2; -import scala.collection.JavaConverters; import scala.runtime.AbstractFunction0; import scala.runtime.AbstractFunction1; import scala.runtime.AbstractFunction2; @@ -141,7 +140,7 @@ public KafkaSystemAdmin(String systemName, Config config, Consumer metadataConsu LOG.info("New admin client with props:" + props); adminClient = AdminClient.create(props); - StreamConfigJava streamConfig = new StreamConfigJava(config); + StreamConfig streamConfig = new StreamConfig(config); KafkaConfig kafkaConfig = new KafkaConfig(streamConfig); coordinatorStreamReplicationFactor = Integer.valueOf(kafkaConfig.getCoordinatorReplicationFactor()); @@ -689,7 +688,7 @@ static Map getIntermediateStreamProperties(Config config) { ApplicationConfig appConfig = new ApplicationConfig(config); if (appConfig.getAppMode() == ApplicationConfig.ApplicationMode.BATCH) { - StreamConfigJava streamConfig = new StreamConfigJava(config); + StreamConfig streamConfig = new StreamConfig(config); intermedidateStreamProperties = streamConfig.getStreamIds() .stream() .filter(streamConfig::getIsIntermediateStream) diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala index b3207f24b6..8d1fd6b8d3 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemFactory.scala @@ -109,7 +109,7 @@ class KafkaSystemFactory extends SystemFactory with Logging { def getIntermediateStreamProperties(config: Config): Map[String, Properties] = { val appConfig = new ApplicationConfig(config) if (appConfig.getAppMode == ApplicationMode.BATCH) { - val streamConfig = new StreamConfigJava(config) + val streamConfig = new StreamConfig(config) streamConfig.getStreamIds().asScala.filter(streamConfig.getIsIntermediateStream(_)).map(streamId => { // only the override here val properties = new Properties() diff --git a/samza-log4j/src/main/java/org/apache/samza/config/Log4jSystemConfig.java b/samza-log4j/src/main/java/org/apache/samza/config/Log4jSystemConfig.java index 8d201367f6..185d83b6e2 100644 --- a/samza-log4j/src/main/java/org/apache/samza/config/Log4jSystemConfig.java +++ b/samza-log4j/src/main/java/org/apache/samza/config/Log4jSystemConfig.java @@ -83,7 +83,7 @@ public String getSerdeClass(String name) { } public String getStreamSerdeName(String systemName, String streamName) { - StreamConfigJava streamConfig = new StreamConfigJava(this); + StreamConfig streamConfig = new StreamConfig(this); Optional option = streamConfig.getStreamMsgSerde(new SystemStream(systemName, streamName)); return option.isPresent() ? option.get() : null; } diff --git a/samza-log4j2/src/main/java/org/apache/samza/config/Log4jSystemConfig.java b/samza-log4j2/src/main/java/org/apache/samza/config/Log4jSystemConfig.java index 8d201367f6..185d83b6e2 100644 --- a/samza-log4j2/src/main/java/org/apache/samza/config/Log4jSystemConfig.java +++ b/samza-log4j2/src/main/java/org/apache/samza/config/Log4jSystemConfig.java @@ -83,7 +83,7 @@ public String getSerdeClass(String name) { } public String getStreamSerdeName(String systemName, String streamName) { - StreamConfigJava streamConfig = new StreamConfigJava(this); + StreamConfig streamConfig = new StreamConfig(this); Optional option = streamConfig.getStreamMsgSerde(new SystemStream(systemName, streamName)); return option.isPresent() ? option.get() : null; } diff --git a/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java b/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java index c0a8a4e8e2..0761f47f7b 100644 --- a/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java +++ b/samza-sql/src/main/java/org/apache/samza/sql/interfaces/SqlIOConfig.java @@ -28,7 +28,7 @@ import org.apache.commons.lang.Validate; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.table.descriptors.CachingTableDescriptor; import org.apache.samza.table.descriptors.RemoteTableDescriptor; import org.apache.samza.table.descriptors.TableDescriptor; @@ -100,11 +100,11 @@ public SqlIOConfig(String systemName, String streamName, List sourcePart if (!isRemoteTable()) { // The below config is required for local table and streams but not for remote table. - streamConfigs.put(String.format(StreamConfigJava.PHYSICAL_NAME_FOR_STREAM_ID, streamId), streamName); + streamConfigs.put(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID, streamId), streamName); if (tableDescriptor != null) { // For local table, set the bootstrap config and default offset to oldest - streamConfigs.put(String.format(StreamConfigJava.BOOTSTRAP_FOR_STREAM_ID, streamId), "true"); - streamConfigs.put(String.format(StreamConfigJava.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID, streamId), "oldest"); + streamConfigs.put(String.format(StreamConfig.BOOTSTRAP_FOR_STREAM_ID, streamId), "true"); + streamConfigs.put(String.format(StreamConfig.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID, streamId), "oldest"); } } diff --git a/samza-sql/src/test/java/org/apache/samza/sql/translator/TestQueryTranslator.java b/samza-sql/src/test/java/org/apache/samza/sql/translator/TestQueryTranslator.java index 89ebad0d48..efe38961c4 100644 --- a/samza-sql/src/test/java/org/apache/samza/sql/translator/TestQueryTranslator.java +++ b/samza-sql/src/test/java/org/apache/samza/sql/translator/TestQueryTranslator.java @@ -29,7 +29,7 @@ import org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.context.ContainerContext; import org.apache.samza.context.Context; import org.apache.samza.operators.OperatorSpecGraph; @@ -83,7 +83,7 @@ public void testTranslate() { translator.translate(queryInfo.get(0), appDesc, 0); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); - StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); + StreamConfig streamConfig = new StreamConfig(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); @@ -122,7 +122,7 @@ public void testTranslateFanIn() { translator.translate(queryInfo.get(1), appDesc, 1); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); - StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); + StreamConfig streamConfig = new StreamConfig(samzaConfig); String inputStreamId1 = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem1 = streamConfig.getSystem(inputStreamId1); String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1); @@ -167,7 +167,7 @@ public void testTranslateFanOut() { translator.translate(queryInfo.get(1), appDesc, 1); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); - StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); + StreamConfig streamConfig = new StreamConfig(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); @@ -211,7 +211,7 @@ public void testTranslateMultiSql() { translator.translate(queryInfo.get(1), appDesc, 1); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); - StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); + StreamConfig streamConfig = new StreamConfig(samzaConfig); String inputStreamId1 = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem1 = streamConfig.getSystem(inputStreamId1); String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1); @@ -259,7 +259,7 @@ public void testTranslateComplex() { translator.translate(queryInfo.get(0), streamAppDesc, 0); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); + StreamConfig streamConfig = new StreamConfig(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); @@ -296,7 +296,7 @@ public void testTranslateSubQuery() { translator.translate(queryInfo.get(0), streamAppDesc, 0); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); + StreamConfig streamConfig = new StreamConfig(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); @@ -631,7 +631,7 @@ public void testTranslateStreamTableInnerJoin() { translator.translate(queryInfo.get(0), streamAppDesc, 0); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); + StreamConfig streamConfig = new StreamConfig(samzaConfig); String input1StreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String input1System = streamConfig.getSystem(input1StreamId); String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId); @@ -699,7 +699,7 @@ public void testTranslateStreamTableLeftJoin() { OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); + StreamConfig streamConfig = new StreamConfig(samzaConfig); String input1StreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String input1System = streamConfig.getSystem(input1StreamId); String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId); @@ -766,7 +766,7 @@ public void testTranslateStreamTableRightJoin() { OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); - StreamConfigJava streamConfig = new StreamConfigJava(samzaConfig); + StreamConfig streamConfig = new StreamConfig(samzaConfig); String input1StreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String input1System = streamConfig.getSystem(input1StreamId); String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId); diff --git a/samza-test/src/main/java/org/apache/samza/test/framework/TestRunner.java b/samza-test/src/main/java/org/apache/samza/test/framework/TestRunner.java index 37b4300aaf..afdbde7951 100644 --- a/samza-test/src/main/java/org/apache/samza/test/framework/TestRunner.java +++ b/samza-test/src/main/java/org/apache/samza/test/framework/TestRunner.java @@ -40,7 +40,7 @@ import org.apache.samza.config.JobConfig; import org.apache.samza.config.JobCoordinatorConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.config.TaskConfig; import org.apache.samza.container.grouper.task.SingleContainerGrouperFactory; import org.apache.samza.context.ExternalContext; @@ -431,9 +431,9 @@ private void deleteDirectory(String path) { * over {@link org.apache.samza.application.descriptors.ApplicationDescriptor} generated configs */ private void addSerdeConfigs(StreamDescriptor descriptor) { - String streamIdPrefix = String.format(StreamConfigJava.STREAM_ID_PREFIX, descriptor.getStreamId()); - String keySerdeConfigKey = streamIdPrefix + StreamConfigJava.KEY_SERDE; - String msgSerdeConfigKey = streamIdPrefix + StreamConfigJava.MSG_SERDE; + String streamIdPrefix = String.format(StreamConfig.STREAM_ID_PREFIX, descriptor.getStreamId()); + String keySerdeConfigKey = streamIdPrefix + StreamConfig.KEY_SERDE; + String msgSerdeConfigKey = streamIdPrefix + StreamConfig.MSG_SERDE; this.configs.put(keySerdeConfigKey, null); this.configs.put(msgSerdeConfigKey, null); } diff --git a/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java b/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java index 03f3894a56..275be34945 100644 --- a/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java +++ b/samza-test/src/test/java/org/apache/samza/test/operator/TestAsyncFlatMap.java @@ -36,7 +36,7 @@ import org.apache.samza.application.descriptors.StreamApplicationDescriptor; import org.apache.samza.config.Config; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.config.TaskConfig; import org.apache.samza.operators.OutputStream; import org.apache.samza.serializers.NoOpSerde; @@ -105,7 +105,7 @@ public void testDownstreamOperatorExceptionIsBubbledUp() { } private List runTest(List pageViews, Map configs) { - configs.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, PAGE_VIEW_STREAM), TEST_SYSTEM); + configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, PAGE_VIEW_STREAM), TEST_SYSTEM); InMemorySystemDescriptor isd = new InMemorySystemDescriptor(TEST_SYSTEM); InMemoryInputDescriptor pageViewStreamDesc = isd diff --git a/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTableWithSideInputsEndToEnd.java b/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTableWithSideInputsEndToEnd.java index b63b81bcfb..eecc6b4dd0 100644 --- a/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTableWithSideInputsEndToEnd.java +++ b/samza-test/src/test/java/org/apache/samza/test/table/TestLocalTableWithSideInputsEndToEnd.java @@ -29,7 +29,7 @@ import org.apache.samza.SamzaException; import org.apache.samza.application.StreamApplication; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfigJava; +import org.apache.samza.config.StreamConfig; import org.apache.samza.application.descriptors.StreamApplicationDescriptor; import org.apache.samza.operators.KV; import org.apache.samza.table.descriptors.TableDescriptor; @@ -82,9 +82,9 @@ public void testJoinWithDurableSideInputTable() { private void runTest(String systemName, StreamApplication app, List pageViews, List profiles) { Map configs = new HashMap<>(); - configs.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, PAGEVIEW_STREAM), systemName); - configs.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, PROFILE_STREAM), systemName); - configs.put(String.format(StreamConfigJava.SYSTEM_FOR_STREAM_ID, ENRICHED_PAGEVIEW_STREAM), systemName); + configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, PAGEVIEW_STREAM), systemName); + configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, PROFILE_STREAM), systemName); + configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, ENRICHED_PAGEVIEW_STREAM), systemName); InMemorySystemDescriptor isd = new InMemorySystemDescriptor(systemName); From 548c9f0c0f553fcfed3898b90de1ad3f7fd7a943 Mon Sep 17 00:00:00 2001 From: Boris S Date: Fri, 27 Sep 2019 17:31:06 -0700 Subject: [PATCH 8/9] review comments --- .../samza/storage/StorageManagerUtil.java | 19 ++- .../storage/TaskSideInputStorageManager.java | 30 ++--- .../samza/checkpoint/OffsetManager.scala | 2 +- .../org/apache/samza/config/StreamConfig.java | 119 +++++++++--------- .../MetricsSnapshotReporterFactory.scala | 1 - .../apache/samza/config/TestStreamConfig.java | 10 +- .../samza/system/kafka/KafkaSystemAdmin.java | 2 +- .../org/apache/samza/config/KafkaConfig.scala | 2 +- 8 files changed, 91 insertions(+), 94 deletions(-) diff --git a/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java b/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java index 9d82b0e66a..a3f971257c 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java +++ b/samza-core/src/main/java/org/apache/samza/storage/StorageManagerUtil.java @@ -20,15 +20,7 @@ package org.apache.samza.storage; import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import java.util.stream.Collectors; import org.apache.samza.clustermanager.StandbyTaskUtil; -import org.apache.samza.config.StreamConfig; import org.apache.samza.container.TaskName; import org.apache.samza.job.model.TaskMode; import org.apache.samza.serializers.model.SamzaObjectMapper; @@ -43,6 +35,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + public class StorageManagerUtil { private static final Logger LOG = LoggerFactory.getLogger(StorageManagerUtil.class); @@ -58,8 +57,8 @@ public class StorageManagerUtil { /** * Fetch the starting offset for the input {@link SystemStreamPartition} * - * Note: The method doesn't respect {@link StreamConfig#CONSUMER_OFFSET_DEFAULT} and - * {@link StreamConfig#CONSUMER_RESET_OFFSET} configurations. It will use the locally + * Note: The method doesn't respect {@link org.apache.samza.config.StreamConfig#CONSUMER_OFFSET_DEFAULT} and + * {@link org.apache.samza.config.StreamConfig#CONSUMER_RESET_OFFSET} configurations. It will use the locally * checkpointed offset if it is valid, or fall back to oldest offset of the stream. * * @param ssp system stream partition for which starting offset is requested diff --git a/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java b/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java index 3b81b2d1eb..1bdd6b7931 100644 --- a/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java +++ b/samza-core/src/main/java/org/apache/samza/storage/TaskSideInputStorageManager.java @@ -20,22 +20,9 @@ package org.apache.samza.storage; import com.google.common.annotations.VisibleForTesting; -import java.io.File; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; import org.apache.samza.Partition; import org.apache.samza.SamzaException; import org.apache.samza.config.Config; -import org.apache.samza.config.StreamConfig; import org.apache.samza.container.TaskName; import org.apache.samza.job.model.TaskMode; import org.apache.samza.storage.kv.Entry; @@ -52,6 +39,19 @@ import org.slf4j.LoggerFactory; import scala.collection.JavaConverters; +import java.io.File; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + /** * A storage manager for all side input stores. It is associated with each {@link org.apache.samza.container.TaskInstance} @@ -163,8 +163,8 @@ public StorageEngine getStore(String storeName) { /** * Gets the starting offset for the given side input {@link SystemStreamPartition}. * - * Note: The method doesn't respect {@link StreamConfig#CONSUMER_OFFSET_DEFAULT} and - * {@link StreamConfig#CONSUMER_RESET_OFFSET} configurations. It will use the local offset + * Note: The method doesn't respect {@link org.apache.samza.config.StreamConfig#CONSUMER_OFFSET_DEFAULT} and + * {@link org.apache.samza.config.StreamConfig#CONSUMER_RESET_OFFSET} configurations. It will use the local offset * file if it is valid, else it will fall back to oldest offset in the stream. * * @param ssp side input system stream partition to get the starting offset for diff --git a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala index 1822c9c659..238280a41a 100644 --- a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala +++ b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala @@ -86,7 +86,7 @@ object OffsetManager extends Logging { case (systemStream, systemStreamMetadata) => // Get default offset. val streamDefaultOffset = streamConfig.getDefaultStreamOffset(systemStream) - val systemDefaultOffset = new SystemConfig(streamConfig).getSystemOffsetDefault(systemStream.getSystem) + val systemDefaultOffset = new SystemConfig(config).getSystemOffsetDefault(systemStream.getSystem) val defaultOffsetType = if (streamDefaultOffset.isPresent) { OffsetType.valueOf(streamDefaultOffset.get.toUpperCase) } else if (systemDefaultOffset != null) { diff --git a/samza-core/src/main/scala/org/apache/samza/config/StreamConfig.java b/samza-core/src/main/scala/org/apache/samza/config/StreamConfig.java index 7477a6ce40..0d8b65d6f8 100644 --- a/samza-core/src/main/scala/org/apache/samza/config/StreamConfig.java +++ b/samza-core/src/main/scala/org/apache/samza/config/StreamConfig.java @@ -52,7 +52,8 @@ public class StreamConfig extends MapConfig { public static final String IS_BOUNDED = SAMZA_PROPERTY + "bounded"; public static final String BROADCAST = SAMZA_PROPERTY + "broadcast"; - // We don't want any external dependencies on these patterns while both exist. Use getProperty to ensure proper values. + // We don't want any external dependencies on these patterns while both exist. + // Use the corresponding get*() method to ensure proper values. private static final String STREAMS_PREFIX = "streams."; public static final String STREAM_PREFIX = "systems.%s.streams.%s."; @@ -165,7 +166,7 @@ private List getStreamIdsForSystem(String system) { * @param systemStream system stream to map to stream id * @return stream id corresponding to the system stream */ - public String systemStreamToStreamId(SystemStream systemStream) { + private String systemStreamToStreamId(SystemStream systemStream) { List streamIds = getStreamIdsForSystem(systemStream.getSystem()).stream() .filter(streamId -> systemStream.getStream().equals(getPhysicalName(streamId))).collect(Collectors.toList()); if (streamIds.size() > 1) { @@ -175,41 +176,6 @@ public String systemStreamToStreamId(SystemStream systemStream) { return streamIds.isEmpty() ? null : streamIds.get(0); } - /** - * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza - * interacts with the stream, as opposed to a property of the stream itself. - * - * First, tries to map the systemStream to a streamId. This will only find a streamId if the stream is a physical name - * (explicitly mapped physical name or a stream id without a physical name mapping). That means this will not map a - * stream id to itself if there is a mapping from the stream id to a physical stream name. This also requires that the - * stream id is mapped to a system in the config. - * If a stream id is found: - * 1) Look for "streams.{streamId}.{property}" for the stream id. - * 2) Otherwise, look for "systems.{systemName}.streams.{streamName}.{property}" in which the systemName is the system - * mapped to the stream id and the streamName is the physical stream name for the stream id. - * 3) Otherwise, look for "systems.{systemName}.default.stream.{property}" in which the systemName is the system - * mapped to the stream id. - * If a stream id was not found or no property could be found using the above keys: - * 1) Look for "systems.{systemName}.streams.{streamName}.{property}" in which the systemName is the system in the - * input systemStream and the streamName is the stream from the input systemStream. - * 2) Otherwise, look for "systems.{systemName}.default.stream.{property}" in which the systemName is the system - * in the input systemStream. - * - * @param systemStream the SystemStream for which the property value will be retrieved. - * @param property the samza property key (including the "samza." prefix); for example, for both - * "streams.streamId.samza.prop.key" and "systems.system.streams.streamName.samza.prop.key", this - * argument should have the value "samza.prop.key" - */ - protected String getSamzaProperty(SystemStream systemStream, String property) { - if (!property.startsWith(SAMZA_PROPERTY)) { - throw new IllegalArgumentException( - String.format("Attempt to fetch a non samza property for SystemStream %s named %s", systemStream, property)); - } - - String streamVal = getAllStreamProperties(systemStreamToStreamId(systemStream)).get(property); - return (streamVal != null) ? streamVal : getSystemStreamProperties(systemStream.getSystem(), systemStream.getStream()).get(property); - } - /** * Gets the System associated with the specified streamId. * It first looks for the property @@ -219,11 +185,11 @@ protected String getSamzaProperty(SystemStream systemStream, String property) { * job.default.system * * @param streamId the identifier for the stream in the config. - * @return the system name associated with the stream. + * @return the system name associated with the stream or null. */ public String getSystem(String streamId) { - return getOrDefault(String.format(SYSTEM_FOR_STREAM_ID, streamId), - new JobConfig(this).getDefaultSystem().orElse(null)); + String system = get(String.format(SYSTEM_FOR_STREAM_ID, streamId)); + return (system != null) ? system : new JobConfig(this).getDefaultSystem().orElse(null); } /** @@ -257,23 +223,6 @@ public boolean getResetOffset(SystemStream systemStream) { return Boolean.valueOf(resetOffset); } - - /** - * Gets a Samza property, with a default value used if no property value is found. - * See getSamzaProperty(SystemStream, String). - * - * @param systemStream the SystemStream for which the property value will be retrieved. - * @param property the samza property key (including the "samza." prefix); for example, for both - * "streams.streamId.samza.prop.key" and "systems.system.streams.streamName.samza.prop.key", this - * argument should have the value "samza.prop.key" - * @param defaultValue the default value to use if the property value is not found - */ - protected String getSamzaProperty(SystemStream systemStream, String property, String defaultValue) { - String streamVal = getSamzaProperty(systemStream, property); - - return streamVal != null ? streamVal : defaultValue; - } - /** * Determines if a Samza property is specified. * See getSamzaProperty(SystemStream, String). @@ -308,7 +257,7 @@ public boolean getBootstrapEnabled(SystemStream systemStream) { } public boolean getBroadcastEnabled(SystemStream systemStream) { - return java.lang.Boolean.parseBoolean(getSamzaProperty(systemStream, BROADCAST)); + return Boolean.parseBoolean(getSamzaProperty(systemStream, BROADCAST)); } public int getPriority(SystemStream systemStream) { @@ -367,7 +316,6 @@ public Config getStreamProperties(String streamId) { return new MapConfig(filteredStreamProperties); } - /** * Gets the boolean flag of whether the specified streamId is an intermediate stream * @@ -391,4 +339,57 @@ public boolean getDeleteCommittedMessages(String streamId) { public boolean getIsBounded(String streamId) { return getBoolean(String.format(IS_BOUNDED_FOR_STREAM_ID, streamId), false); } + + /** + * Gets the specified Samza property for a SystemStream. A Samza property is a property that controls how Samza + * interacts with the stream, as opposed to a property of the stream itself. + * + * First, tries to map the systemStream to a streamId. This will only find a streamId if the stream is a physical name + * (explicitly mapped physical name or a stream id without a physical name mapping). That means this will not map a + * stream id to itself if there is a mapping from the stream id to a physical stream name. This also requires that the + * stream id is mapped to a system in the config. + * If a stream id is found: + * 1) Look for "streams.{streamId}.{property}" for the stream id. + * 2) Otherwise, look for "systems.{systemName}.streams.{streamName}.{property}" in which the systemName is the system + * mapped to the stream id and the streamName is the physical stream name for the stream id. + * 3) Otherwise, look for "systems.{systemName}.default.stream.{property}" in which the systemName is the system + * mapped to the stream id. + * If a stream id was not found or no property could be found using the above keys: + * 1) Look for "systems.{systemName}.streams.{streamName}.{property}" in which the systemName is the system in the + * input systemStream and the streamName is the stream from the input systemStream. + * 2) Otherwise, look for "systems.{systemName}.default.stream.{property}" in which the systemName is the system + * in the input systemStream. + * 3) or null + * + * @param systemStream the SystemStream for which the property value will be retrieved. + * @param property the samza property key (including the "samza." prefix); for example, for both + * "streams.streamId.samza.prop.key" and "systems.system.streams.streamName.samza.prop.key", this + * argument should have the value "samza.prop.key" + * @return the property value + */ + private String getSamzaProperty(SystemStream systemStream, String property) { + if (!property.startsWith(SAMZA_PROPERTY)) { + throw new IllegalArgumentException( + String.format("Attempt to fetch a non samza property for SystemStream %s named %s", systemStream, property)); + } + + String streamVal = getAllStreamProperties(systemStreamToStreamId(systemStream)).get(property); + return (streamVal != null) ? streamVal : getSystemStreamProperties(systemStream.getSystem(), systemStream.getStream()).get(property); + } + + /** + * Gets a Samza property, with a default value used if no property value is found. + * See getSamzaProperty(SystemStream, String). + * + * @param systemStream the SystemStream for which the property value will be retrieved. + * @param property the samza property key (including the "samza." prefix); for example, for both + * "streams.streamId.samza.prop.key" and "systems.system.streams.streamName.samza.prop.key", this + * argument should have the value "samza.prop.key" + * @param defaultValue the default value to use if the property value is not found + */ + private String getSamzaProperty(SystemStream systemStream, String property, String defaultValue) { + String streamVal = getSamzaProperty(systemStream, property); + + return streamVal != null ? streamVal : defaultValue; + } } diff --git a/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala b/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala index e3e33d5c80..ec5f4d971f 100644 --- a/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala +++ b/samza-core/src/main/scala/org/apache/samza/metrics/reporter/MetricsSnapshotReporterFactory.scala @@ -22,7 +22,6 @@ package org.apache.samza.metrics.reporter import org.apache.samza.util.{Logging, StreamUtil, Util} import org.apache.samza.SamzaException import org.apache.samza.config.{Config, JobConfig, MetricsConfig, SerializerConfig, StreamConfig, SystemConfig} -import StreamConfig import org.apache.samza.metrics.MetricsReporter import org.apache.samza.metrics.MetricsReporterFactory import org.apache.samza.metrics.MetricsRegistryMap diff --git a/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java b/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java index 9d02393f18..bbea19e528 100644 --- a/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java +++ b/samza-core/src/test/java/org/apache/samza/config/TestStreamConfig.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.apache.samza.system.SystemStream; import org.junit.Test; @@ -474,13 +475,11 @@ public void testGetStreamIds() { StreamConfig streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + ".property", "value"))); - assertEquals(ImmutableList.of(STREAM_ID), - ImmutableList.copyOf(streamConfig.getStreamIds())); + assertEquals(ImmutableSet.of(STREAM_ID), ImmutableSet.copyOf(streamConfig.getStreamIds())); streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + ".property.subProperty", "value"))); - assertEquals(ImmutableList.of(STREAM_ID), - ImmutableList.copyOf(streamConfig.getStreamIds())); + assertEquals(ImmutableSet.of(STREAM_ID), ImmutableSet.copyOf(streamConfig.getStreamIds())); streamConfig = new StreamConfig(new MapConfig(ImmutableMap.of( String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + ".property0", "value", @@ -488,8 +487,7 @@ public void testGetStreamIds() { String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + ".property.subProperty0", "value", String.format(StreamConfig.STREAM_ID_PREFIX, STREAM_ID) + ".property.subProperty1", "value", String.format(StreamConfig.STREAM_ID_PREFIX, OTHER_STREAM_ID) + ".property", "value"))); - assertEquals(ImmutableList.of(STREAM_ID, OTHER_STREAM_ID), - ImmutableList.copyOf(streamConfig.getStreamIds())); + assertEquals(ImmutableSet.of(STREAM_ID, OTHER_STREAM_ID), ImmutableSet.copyOf(streamConfig.getStreamIds())); } private static void doTestSamzaProperty(String propertyName, String propertyValue, SamzaPropertyAssertion assertion) { diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index b6c9ba1e86..f0bce193d9 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -142,7 +142,7 @@ public KafkaSystemAdmin(String systemName, Config config, Consumer metadataConsu StreamConfig streamConfig = new StreamConfig(config); - KafkaConfig kafkaConfig = new KafkaConfig(streamConfig); + KafkaConfig kafkaConfig = new KafkaConfig(config); coordinatorStreamReplicationFactor = Integer.valueOf(kafkaConfig.getCoordinatorReplicationFactor()); coordinatorStreamProperties = getCoordinatorStreamProperties(kafkaConfig); diff --git a/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala b/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala index 1261d055b5..f8051f27f1 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala @@ -290,7 +290,7 @@ class KafkaConfig(config: Config) extends ScalaMapConfig(config) { } // The method returns a map of storenames to changelog topic names, which are configured to use kafka as the changelog stream - def getKafkaChangelogEnabledStores() = { + def getKafkaChangelogEnabledStores(): util.HashMap[String, String] = { val changelogConfigs = config.regexSubset(KafkaConfig.CHANGELOG_STREAM_NAMES_REGEX).asScala //var storeToChangelog = Map[String, String]() var storeToChangelog :util.HashMap[String, String] = new util.HashMap(); From 20f85080fab47206a5cb1b26fd54f9f6065f50d0 Mon Sep 17 00:00:00 2001 From: Boris S Date: Mon, 30 Sep 2019 10:00:01 -0700 Subject: [PATCH 9/9] modified comment --- .../src/main/scala/org/apache/samza/config/StreamConfig.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samza-core/src/main/scala/org/apache/samza/config/StreamConfig.java b/samza-core/src/main/scala/org/apache/samza/config/StreamConfig.java index 0d8b65d6f8..8ee044e85c 100644 --- a/samza-core/src/main/scala/org/apache/samza/config/StreamConfig.java +++ b/samza-core/src/main/scala/org/apache/samza/config/StreamConfig.java @@ -68,7 +68,8 @@ public class StreamConfig extends MapConfig { public static final String BOOTSTRAP_FOR_STREAM_ID = STREAM_ID_PREFIX + BOOTSTRAP; public static final String BROADCAST_FOR_STREAM_ID = STREAM_ID_PREFIX + BROADCAST; - /** + /* + * Implementation notes: * Helper for accessing configs related to stream properties. * * For most configs, this currently supports two different formats for specifying stream properties: