From a31a7aa29b7be4bb46f8e651b6b8fa46a65b48e2 Mon Sep 17 00:00:00 2001 From: Boris Shkolnik Date: Mon, 16 Oct 2017 15:25:49 -0700 Subject: [PATCH 01/18] reduce debugging from info to debug in KafkaCheckpointManager.java --- .../apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samza-kafka/src/main/scala/org/apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala b/samza-kafka/src/main/scala/org/apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala index 4eb66667c2..b016b4a980 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/checkpoint/kafka/KafkaCheckpointManager.scala @@ -103,7 +103,7 @@ class KafkaCheckpointManager( systemProducer.send(taskName.getTaskName, envelope) systemProducer.flush(taskName.getTaskName) // make sure it is written - info("Completed writing checkpoint=%s into %s topic for system %s." format(checkpoint, checkpointTopic, systemName) ) + debug("Completed writing checkpoint=%s into %s topic for system %s." format(checkpoint, checkpointTopic, systemName) ) loop.done }, From 824e19ae966eee15e3265bdea828e9c36fd45383 Mon Sep 17 00:00:00 2001 From: Boris S Date: Wed, 17 Oct 2018 12:24:39 -0700 Subject: [PATCH 02/18] restore deprecated kafka consumer classes --- .../DefaultFetchSimpleConsumer.scala | 66 ++ .../system/kafka_deprecated/GetOffset.scala | 116 ++++ .../kafka_deprecated/KafkaConsumerProxy.java | 423 ++++++++++++ .../kafka_deprecated/KafkaSystemAdmin.scala | 609 ++++++++++++++++++ .../kafka_deprecated/KafkaSystemConsumer.java | 371 +++++++++++ .../KafkaSystemConsumerMetrics.scala | 96 +++ .../kafka_deprecated/KafkaSystemFactory.scala | 166 +++++ .../KafkaSystemProducer.scala | 235 +++++++ .../KafkaSystemProducerMetrics.scala | 42 ++ .../system/kafka_deprecated/MessageSink.scala | 35 + .../kafka_deprecated/TopicMetadataCache.scala | 78 +++ .../samza/system/kafka_deprecated/Toss.scala | 28 + 12 files changed, 2265 insertions(+) create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/DefaultFetchSimpleConsumer.scala create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/GetOffset.scala create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaConsumerProxy.java create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemAdmin.scala create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.java create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumerMetrics.scala create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemProducer.scala create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemProducerMetrics.scala create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/MessageSink.scala create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/TopicMetadataCache.scala create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/Toss.scala diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/DefaultFetchSimpleConsumer.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/DefaultFetchSimpleConsumer.scala new file mode 100644 index 0000000000..5f79ea568a --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/DefaultFetchSimpleConsumer.scala @@ -0,0 +1,66 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.samza.system.kafka_deprecated + +import kafka.consumer.SimpleConsumer +import kafka.api._ +import kafka.common.TopicAndPartition +import kafka.consumer.ConsumerConfig + +class DefaultFetchSimpleConsumer(host: scala.Predef.String, port: scala.Int, soTimeout: scala.Int, bufferSize: scala.Int, + clientId: scala.Predef.String, fetchSize: StreamFetchSizes = new StreamFetchSizes, + minBytes: Int = ConsumerConfig.MinFetchBytes, maxWait: Int = ConsumerConfig.MaxFetchWaitMs) + extends SimpleConsumer(host, port, soTimeout, bufferSize, clientId) { + + def defaultFetch(fetches: (TopicAndPartition, Long)*) = { + val fbr = new FetchRequestBuilder().maxWait(maxWait) + .minBytes(minBytes) + .clientId(clientId) + + fetches.foreach(f => fbr.addFetch(f._1.topic, f._1.partition, f._2, fetchSize.streamValue.getOrElse(f._1.topic, fetchSize.defaultValue))) + + this.fetch(fbr.build()) + } + + override def close(): Unit = super.close() + + override def send(request: TopicMetadataRequest): TopicMetadataResponse = super.send(request) + + override def fetch(request: FetchRequest): FetchResponse = super.fetch(request) + + override def getOffsetsBefore(request: OffsetRequest): OffsetResponse = super.getOffsetsBefore(request) + + override def commitOffsets(request: OffsetCommitRequest): OffsetCommitResponse = super.commitOffsets(request) + + override def fetchOffsets(request: OffsetFetchRequest): OffsetFetchResponse = super.fetchOffsets(request) + + override def earliestOrLatestOffset(topicAndPartition: TopicAndPartition, earliestOrLatest: Long, consumerId: Int): Long = super.earliestOrLatestOffset(topicAndPartition, earliestOrLatest, consumerId) +} + +/** + * a simple class for holding values for the stream's fetch size (fetch.message.max.bytes). + * The stream-level fetch size values are put in the streamValue map streamName -> fetchSize. + * If stream-level fetch size is not defined, use the default value. The default value is the + * Kafka's default fetch size value or the system-level fetch size value (if defined). + */ +case class StreamFetchSizes(defaultValue: Int = ConsumerConfig.MaxFetchSize, streamValue: Map[String, Int] = Map[String, Int]()) + diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/GetOffset.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/GetOffset.scala new file mode 100644 index 0000000000..c4e7354dc3 --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/GetOffset.scala @@ -0,0 +1,116 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.samza.system.kafka_deprecated + +import org.apache.kafka.common.errors.OffsetOutOfRangeException +import kafka.api._ +import kafka.common.TopicAndPartition +import kafka.api.PartitionOffsetRequestInfo +import org.apache.samza.util.Logging +import org.apache.samza.util.KafkaUtil + +/** + * GetOffset validates offsets for topic partitions, and manages fetching new + * offsets for topics using Kafka's auto.offset.reset configuration. + */ +class GetOffset( + /** + * The default auto.offset.reset to use if a topic is not overridden in + * autoOffsetResetTopics. Any value other than "earliest" or "latest" will + * result in an exception when getRestOffset is called. + */ + default: String, + + /** + * Topic-level overrides for auto.offset.reset. Any value other than + * "earliest" or "latest" will result in an exception when getRestOffset is + * called. + */ + autoOffsetResetTopics: Map[String, String] = Map()) extends Logging with Toss { + + /** + * Checks if an offset is valid for a given topic/partition. Validity is + * defined as an offset that returns a readable non-empty message set with + * no exceptions. + */ + def isValidOffset(consumer: DefaultFetchSimpleConsumer, topicAndPartition: TopicAndPartition, offset: String) = { + info("Validating offset %s for topic and partition %s" format (offset, topicAndPartition)) + + try { + val messages = consumer.defaultFetch((topicAndPartition, offset.toLong)) + + if (messages.hasError) { + KafkaUtil.maybeThrowException(messages.error(topicAndPartition.topic, topicAndPartition.partition).exception()) + } + + info("Able to successfully read from offset %s for topic and partition %s. Using it to instantiate consumer." format (offset, topicAndPartition)) + + true + } catch { + case e: OffsetOutOfRangeException => false + } + } + + /** + * Uses a topic's auto.offset.reset setting (defined via the + * autoOffsetResetTopics map in the constructor) to fetch either the + * earliest or latest offset. If neither earliest or latest is defined for + * the topic in question, the default supplied in the constructor will be + * used. + */ + def getResetOffset(consumer: DefaultFetchSimpleConsumer, topicAndPartition: TopicAndPartition) = { + val offsetRequest = new OffsetRequest(Map(topicAndPartition -> new PartitionOffsetRequestInfo(getAutoOffset(topicAndPartition.topic), 1))) + val offsetResponse = consumer.getOffsetsBefore(offsetRequest) + val partitionOffsetResponse = offsetResponse + .partitionErrorAndOffsets + .get(topicAndPartition) + .getOrElse(toss("Unable to find offset information for %s" format topicAndPartition)) + + KafkaUtil.maybeThrowException(partitionOffsetResponse.error.exception()) + + partitionOffsetResponse + .offsets + .headOption + .getOrElse(toss("Got response, but no offsets defined for %s" format topicAndPartition)) + } + + /** + * Returns either the earliest or latest setting (a Kafka constant) for a + * given topic using the autoOffsetResetTopics map defined in the + * constructor. If the topic is not defined in autoOffsetResetTopics, the + * default value supplied in the constructor will be used. This is used in + * conjunction with getResetOffset to fetch either the earliest or latest + * offset for a topic. + */ + private def getAutoOffset(topic: String): Long = { + info("Checking if auto.offset.reset is defined for topic %s" format (topic)) + autoOffsetResetTopics.getOrElse(topic, default) match { + case OffsetRequest.LargestTimeString => + info("Got reset of type %s." format OffsetRequest.LargestTimeString) + OffsetRequest.LatestTime + case OffsetRequest.SmallestTimeString => + info("Got reset of type %s." format OffsetRequest.SmallestTimeString) + OffsetRequest.EarliestTime + case other => toss("Can't get offset value for topic %s due to invalid value: %s" format (topic, other)) + } + } +} diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaConsumerProxy.java b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaConsumerProxy.java new file mode 100644 index 0000000000..5ace182f77 --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaConsumerProxy.java @@ -0,0 +1,423 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.samza.system.kafka_deprecated; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import kafka.common.TopicAndPartition; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.common.Metric; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.TopicPartition; +import org.apache.samza.Partition; +import org.apache.samza.SamzaException; +import org.apache.samza.system.IncomingMessageEnvelope; +import org.apache.samza.system.SystemStreamPartition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class contains a separate thread that reads messages from kafka and puts them into the BlockingEnvelopeMap + * through KafkaSystemConsumer.KafkaConsumerMessageSink object. + * This class is not thread safe. There will be only one instance of this class per KafkaSystemConsumer object. + * We still need some synchronization around kafkaConsumer. See pollConsumer() method for details. + */ +class KafkaConsumerProxy { + private static final Logger LOG = LoggerFactory.getLogger(KafkaConsumerProxy.class); + + private static final int SLEEP_MS_WHILE_NO_TOPIC_PARTITION = 100; + + final Thread consumerPollThread; + private final Consumer kafkaConsumer; + private final KafkaSystemConsumer.KafkaConsumerMessageSink sink; + private final KafkaSystemConsumerMetrics kafkaConsumerMetrics; + private final String metricName; + private final String systemName; + private final String clientId; + private final Map topicPartitionToSSP = new HashMap<>(); + private final Map perPartitionMetrics = new HashMap<>(); + // list of all the SSPs we poll from, with their next(most recently read + 1) offsets correspondingly. + private final Map nextOffsets = new ConcurrentHashMap<>(); + // lags behind the high water mark, as reported by the Kafka consumer. + private final Map latestLags = new HashMap<>(); + + private volatile boolean isRunning = false; + private volatile Throwable failureCause = null; + private final CountDownLatch consumerPollThreadStartLatch = new CountDownLatch(1); + + KafkaConsumerProxy(Consumer kafkaConsumer, String systemName, String clientId, + KafkaSystemConsumer.KafkaConsumerMessageSink messageSink, KafkaSystemConsumerMetrics samzaConsumerMetrics, + String metricName) { + + this.kafkaConsumer = kafkaConsumer; + this.systemName = systemName; + this.sink = messageSink; + this.kafkaConsumerMetrics = samzaConsumerMetrics; + this.metricName = metricName; + this.clientId = clientId; + + this.kafkaConsumerMetrics.registerClientProxy(metricName); + + consumerPollThread = new Thread(createProxyThreadRunnable()); + consumerPollThread.setDaemon(true); + consumerPollThread.setName( + "Samza KafkaConsumerProxy Poll " + consumerPollThread.getName() + " - " + systemName); + } + + /** + * Add new partition to the list of polled partitions. + * Bust only be called before {@link KafkaConsumerProxy#start} is called.. + */ + public void addTopicPartition(SystemStreamPartition ssp, long nextOffset) { + LOG.info(String.format("Adding new topicPartition %s with offset %s to queue for consumer %s", ssp, nextOffset, + this)); + topicPartitionToSSP.put(KafkaSystemConsumer.toTopicPartition(ssp), ssp); //registered SSPs + + // this is already vetted offset so there is no need to validate it + nextOffsets.put(ssp, nextOffset); + + kafkaConsumerMetrics.setNumTopicPartitions(metricName, nextOffsets.size()); + } + + /** + * Stop this KafkaConsumerProxy and wait for at most {@code timeoutMs}. + * @param timeoutMs maximum time to wait to stop this KafkaConsumerProxy + */ + public void stop(long timeoutMs) { + LOG.info("Shutting down KafkaConsumerProxy poll thread {} for {}", consumerPollThread.getName(), this); + + isRunning = false; + try { + consumerPollThread.join(timeoutMs/2); + // join() may timeout + // in this case we should interrupt it and wait again + if (consumerPollThread.isAlive()) { + consumerPollThread.interrupt(); + consumerPollThread.join(timeoutMs/2); + } + } catch (InterruptedException e) { + LOG.warn("Join in KafkaConsumerProxy has failed", e); + consumerPollThread.interrupt(); + } + } + + public void start() { + if (!consumerPollThread.isAlive()) { + LOG.info("Starting KafkaConsumerProxy polling thread for " + this.toString()); + + consumerPollThread.start(); + + // we need to wait until the thread starts + while (!isRunning && failureCause == null) { + try { + consumerPollThreadStartLatch.await(3000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + LOG.info("Ignoring InterruptedException while waiting for consumer poll thread to start.", e); + } + } + } else { + LOG.warn("Tried to start an already started KafkaConsumerProxy (%s). Ignoring.", this.toString()); + } + + if (topicPartitionToSSP.size() == 0) { + String msg = String.format("Cannot start KafkaConsumerProxy without any registered TopicPartitions for %s", systemName); + LOG.error(msg); + throw new SamzaException(msg); + } + } + + boolean isRunning() { + return isRunning; + } + + Throwable getFailureCause() { + return failureCause; + } + + private void initializeLags() { + // This is expensive, so only do it once at the beginning. After the first poll, we can rely on metrics for lag. + Map endOffsets = kafkaConsumer.endOffsets(topicPartitionToSSP.keySet()); + endOffsets.forEach((tp, offset) -> { + SystemStreamPartition ssp = topicPartitionToSSP.get(tp); + long startingOffset = nextOffsets.get(ssp); + // End offsets are the offset of the newest message + 1 + // If the message we are about to consume is < end offset, we are starting with a lag. + long initialLag = endOffsets.get(tp) - startingOffset; + + LOG.info("Initial lag for SSP {} is {} (end={}, startOffset={})", ssp, initialLag, endOffsets.get(tp), startingOffset); + latestLags.put(ssp, initialLag); + sink.setIsAtHighWatermark(ssp, initialLag == 0); + }); + + // initialize lag metrics + refreshLagMetrics(); + } + + // creates a separate thread for getting the messages. + private Runnable createProxyThreadRunnable() { + Runnable runnable = () -> { + isRunning = true; + + try { + consumerPollThreadStartLatch.countDown(); + LOG.info("Starting consumer poll thread {} for system {}", consumerPollThread.getName(), systemName); + initializeLags(); + while (isRunning) { + fetchMessages(); + } + } catch (Throwable throwable) { + LOG.error(String.format("Error in KafkaConsumerProxy poll thread for system: %s.", systemName), throwable); + // KafkaSystemConsumer uses the failureCause to propagate the throwable to the container + failureCause = throwable; + isRunning = false; + } + + if (!isRunning) { + LOG.info("KafkaConsumerProxy for system {} has stopped.", systemName); + } + }; + + return runnable; + } + + private void fetchMessages() { + Set sspsToFetch = new HashSet<>(); + for (SystemStreamPartition ssp : nextOffsets.keySet()) { + if (sink.needsMoreMessages(ssp)) { + sspsToFetch.add(ssp); + } + } + LOG.debug("pollConsumer for {} SSPs: {}", sspsToFetch.size(), sspsToFetch); + if (!sspsToFetch.isEmpty()) { + kafkaConsumerMetrics.incClientReads(metricName); + + Map> response; + + response = pollConsumer(sspsToFetch, 500L); + + // move the responses into the queue + for (Map.Entry> e : response.entrySet()) { + List envelopes = e.getValue(); + if (envelopes != null) { + moveMessagesToTheirQueue(e.getKey(), envelopes); + } + } + + populateCurrentLags(sspsToFetch); // find current lags for for each SSP + } else { // nothing to read + + LOG.debug("No topic/partitions need to be fetched for system {} right now. Sleeping {}ms.", systemName, + SLEEP_MS_WHILE_NO_TOPIC_PARTITION); + + kafkaConsumerMetrics.incClientSkippedFetchRequests(metricName); + + try { + Thread.sleep(SLEEP_MS_WHILE_NO_TOPIC_PARTITION); + } catch (InterruptedException e) { + LOG.warn("Sleep in fetchMessages was interrupted"); + } + } + refreshLagMetrics(); + } + + // the actual polling of the messages from kafka + private Map> pollConsumer( + Set systemStreamPartitions, long timeoutMs) { + + // Since we need to poll only from some subset of TopicPartitions (passed as the argument), + // we need to pause the rest. + List topicPartitionsToPause = new ArrayList<>(); + List topicPartitionsToPoll = new ArrayList<>(); + + for (Map.Entry e : topicPartitionToSSP.entrySet()) { + TopicPartition tp = e.getKey(); + SystemStreamPartition ssp = e.getValue(); + if (systemStreamPartitions.contains(ssp)) { + topicPartitionsToPoll.add(tp); // consume + } else { + topicPartitionsToPause.add(tp); // ignore + } + } + + ConsumerRecords records; + try { + // Synchronize, in case the consumer is used in some other thread (metadata or something else) + synchronized (kafkaConsumer) { + // Since we are not polling from ALL the subscribed topics, so we need to "change" the subscription temporarily + kafkaConsumer.pause(topicPartitionsToPause); + kafkaConsumer.resume(topicPartitionsToPoll); + records = kafkaConsumer.poll(timeoutMs); + } + } catch (Exception e) { + // we may get InvalidOffsetException | AuthorizationException | KafkaException exceptions, + // but we still just rethrow, and log it up the stack. + LOG.error("Caught a Kafka exception in pollConsumer for system " + systemName, e); + throw e; + } + + return processResults(records); + } + + private Map> processResults(ConsumerRecords records) { + if (records == null) { + throw new SamzaException("Received null 'records' after polling consumer in KafkaConsumerProxy " + this); + } + + Map> results = new HashMap<>(records.count()); + // Parse the returned records and convert them into the IncomingMessageEnvelope. + for (ConsumerRecord record : records) { + int partition = record.partition(); + String topic = record.topic(); + TopicPartition tp = new TopicPartition(topic, partition); + + updateMetrics(record, tp); + + SystemStreamPartition ssp = topicPartitionToSSP.get(tp); + List messages = results.get(ssp); + if (messages == null) { + messages = new ArrayList<>(); + results.put(ssp, messages); + } + + K key = record.key(); + Object value = record.value(); + IncomingMessageEnvelope imEnvelope = + new IncomingMessageEnvelope(ssp, String.valueOf(record.offset()), key, value, getRecordSize(record)); + messages.add(imEnvelope); + } + if (LOG.isDebugEnabled()) { + LOG.debug("# records per SSP:"); + for (Map.Entry> e : results.entrySet()) { + List list = e.getValue(); + LOG.debug(e.getKey() + " = " + ((list == null) ? 0 : list.size())); + } + } + + return results; + } + + private int getRecordSize(ConsumerRecord r) { + int keySize = (r.key() == null) ? 0 : r.serializedKeySize(); + return keySize + r.serializedValueSize(); + } + + private void updateMetrics(ConsumerRecord r, TopicPartition tp) { + TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp); + SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition())); + + Long lag = latestLags.get(ssp); + if (lag == null) { + throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName); + } + long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark + if (currentSSPLag < 0) { + return; + } + + long recordOffset = r.offset(); + long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark + + int size = getRecordSize(r); + kafkaConsumerMetrics.incReads(tap); + kafkaConsumerMetrics.incBytesReads(tap, size); + kafkaConsumerMetrics.setOffsets(tap, recordOffset); + kafkaConsumerMetrics.incClientBytesReads(metricName, size); + kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark); + } + + private void moveMessagesToTheirQueue(SystemStreamPartition ssp, List envelopes) { + long nextOffset = nextOffsets.get(ssp); + + for (IncomingMessageEnvelope env : envelopes) { + sink.addMessage(ssp, env); // move message to the BlockingEnvelopeMap's queue + + LOG.trace("IncomingMessageEnvelope. got envelope with offset:{} for ssp={}", env.getOffset(), ssp); + nextOffset = Long.valueOf(env.getOffset()) + 1; + } + + nextOffsets.put(ssp, nextOffset); + } + + // The only way to figure out lag for the KafkaConsumer is to look at the metrics after each poll() call. + // One of the metrics (records-lag) shows how far behind the HighWatermark the consumer is. + // This method populates the lag information for each SSP into latestLags member variable. + private void populateCurrentLags(Set ssps) { + + Map consumerMetrics = kafkaConsumer.metrics(); + + // populate the MetricNames first time + if (perPartitionMetrics.isEmpty()) { + HashMap tags = new HashMap<>(); + tags.put("client-id", clientId); // this is required by the KafkaConsumer to get the metrics + + for (SystemStreamPartition ssp : ssps) { + TopicPartition tp = KafkaSystemConsumer.toTopicPartition(ssp); + perPartitionMetrics.put(ssp, new MetricName(tp + ".records-lag", "consumer-fetch-manager-metrics", "", tags)); + } + } + + for (SystemStreamPartition ssp : ssps) { + MetricName mn = perPartitionMetrics.get(ssp); + Metric currentLagMetric = consumerMetrics.get(mn); + + // High watermark is fixed to be the offset of last available message, + // so the lag is now at least 0, which is the same as Samza's definition. + // If the lag is not 0, then isAtHead is not true, and kafkaClient keeps polling. + long currentLag = (currentLagMetric != null) ? (long) currentLagMetric.value() : -1L; + latestLags.put(ssp, currentLag); + + // calls the setIsAtHead for the BlockingEnvelopeMap + sink.setIsAtHighWatermark(ssp, currentLag == 0); + } + } + + private void refreshLagMetrics() { + for (Map.Entry e : nextOffsets.entrySet()) { + SystemStreamPartition ssp = e.getKey(); + Long offset = e.getValue(); + TopicAndPartition tp = new TopicAndPartition(ssp.getStream(), ssp.getPartition().getPartitionId()); + Long lag = latestLags.get(ssp); + LOG.trace("Latest offset of {} is {}; lag = {}", ssp, offset, lag); + if (lag != null && offset != null && lag >= 0) { + long streamEndOffset = offset.longValue() + lag.longValue(); + // update the metrics + kafkaConsumerMetrics.setHighWatermarkValue(tp, streamEndOffset); + kafkaConsumerMetrics.setLagValue(tp, lag.longValue()); + } + } + } + + @Override + public String toString() { + return String.format("consumerProxy-%s-%s", systemName, clientId); + } +} + diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemAdmin.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemAdmin.scala new file mode 100644 index 0000000000..e7ff7496be --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemAdmin.scala @@ -0,0 +1,609 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.system.kafka_deprecated + +import java.util +import java.util.{Properties, UUID} + +import com.google.common.annotations.VisibleForTesting +import kafka.admin.{AdminClient, AdminUtils} +import kafka.api._ +import kafka.common.TopicAndPartition +import kafka.consumer.{ConsumerConfig, SimpleConsumer} +import kafka.utils.ZkUtils +import org.apache.kafka.clients.CommonClientConfigs +import org.apache.kafka.common.errors.TopicExistsException +import org.apache.kafka.common.TopicPartition +import org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata +import org.apache.samza.system._ +import org.apache.samza.system.kafka.KafkaStreamSpec +import org.apache.samza.util.{ClientUtilTopicMetadataStore, ExponentialSleepStrategy, KafkaUtil, Logging} +import org.apache.samza.{Partition, SamzaException} + +import scala.collection.JavaConverters._ + + +object KafkaSystemAdmin extends Logging { + + @VisibleForTesting @volatile var deleteMessagesCalled = false + val CLEAR_STREAM_RETRIES = 3 + + /** + * A helper method that takes oldest, newest, and upcoming offsets for each + * system stream partition, and creates a single map from stream name to + * SystemStreamMetadata. + */ + def assembleMetadata(oldestOffsets: Map[SystemStreamPartition, String], newestOffsets: Map[SystemStreamPartition, String], upcomingOffsets: Map[SystemStreamPartition, String]): Map[String, SystemStreamMetadata] = { + val allMetadata = (oldestOffsets.keySet ++ newestOffsets.keySet ++ upcomingOffsets.keySet) + .groupBy(_.getStream) + .map { + case (streamName, systemStreamPartitions) => + val streamPartitionMetadata = systemStreamPartitions + .map(systemStreamPartition => { + val partitionMetadata = new SystemStreamPartitionMetadata( + // If the topic/partition is empty then oldest and newest will + // be stripped of their offsets, so default to null. + oldestOffsets.getOrElse(systemStreamPartition, null), + newestOffsets.getOrElse(systemStreamPartition, null), + upcomingOffsets(systemStreamPartition)) + (systemStreamPartition.getPartition, partitionMetadata) + }) + .toMap + val streamMetadata = new SystemStreamMetadata(streamName, streamPartitionMetadata.asJava) + (streamName, streamMetadata) + } + .toMap + + // This is typically printed downstream and it can be spammy, so debug level here. + debug("Got metadata: %s" format allMetadata) + + allMetadata + } +} + +/** + * A helper class that is used to construct the changelog stream specific information + * + * @param replicationFactor The number of replicas for the changelog stream + * @param kafkaProps The kafka specific properties that need to be used for changelog stream creation + */ +case class ChangelogInfo(var replicationFactor: Int, var kafkaProps: Properties) + +/** + * A Kafka-based implementation of SystemAdmin. + */ +class KafkaSystemAdmin( + /** + * The system name to use when creating SystemStreamPartitions to return in + * the getSystemStreamMetadata responser. + */ + systemName: String, + + // TODO whenever Kafka decides to make the Set[Broker] class public, let's switch to Set[Broker] here. + /** + * List of brokers that are part of the Kafka system that we wish to + * interact with. The format is host1:port1,host2:port2. + */ + brokerListString: String, + + /** + * A method that returns a ZkUtils for the Kafka system. This is invoked + * when the system admin is attempting to create a coordinator stream. + */ + connectZk: () => ZkUtils, + + /** + * Custom properties to use when the system admin tries to create a new + * coordinator stream. + */ + coordinatorStreamProperties: Properties = new Properties, + + /** + * The replication factor to use when the system admin creates a new + * coordinator stream. + */ + coordinatorStreamReplicationFactor: Int = 1, + + /** + * The timeout to use for the simple consumer when fetching metadata from + * Kafka. Equivalent to Kafka's socket.timeout.ms configuration. + */ + timeout: Int = Int.MaxValue, + + /** + * The buffer size to use for the simple consumer when fetching metadata + * from Kafka. Equivalent to Kafka's socket.receive.buffer.bytes + * configuration. + */ + bufferSize: Int = ConsumerConfig.SocketBufferSize, + + /** + * The client ID to use for the simple consumer when fetching metadata from + * Kafka. Equivalent to Kafka's client.id configuration. + */ + clientId: String = UUID.randomUUID.toString, + + /** + * Replication factor for the Changelog topic in kafka + * Kafka properties to be used during the Changelog topic creation + */ + topicMetaInformation: Map[String, ChangelogInfo] = Map[String, ChangelogInfo](), + + /** + * Kafka properties to be used during the intermediate topic creation + */ + intermediateStreamProperties: Map[String, Properties] = Map(), + + /** + * Whether deleteMessages() API can be used + */ + deleteCommittedMessages: Boolean = false) extends ExtendedSystemAdmin with Logging { + + import KafkaSystemAdmin._ + + @volatile var running = false + @volatile var adminClient: AdminClient = null + + override def start() = { + if (!running) { + running = true + adminClient = createAdminClient() + } + } + + override def stop() = { + if (running) { + running = false + adminClient.close() + adminClient = null + } + } + + private def createAdminClient(): AdminClient = { + val props = new Properties() + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerListString) + AdminClient.create(props) + } + + override def getSystemStreamPartitionCounts(streams: util.Set[String], cacheTTL: Long): util.Map[String, SystemStreamMetadata] = { + getSystemStreamPartitionCounts(streams, new ExponentialSleepStrategy(initialDelayMs = 500), cacheTTL) + } + + def getSystemStreamPartitionCounts(streams: util.Set[String], retryBackoff: ExponentialSleepStrategy, cacheTTL: Long = Long.MaxValue): util.Map[String, SystemStreamMetadata] = { + debug("Fetching system stream partition count for: %s" format streams) + var metadataTTL = cacheTTL + retryBackoff.run( + loop => { + val metadata = TopicMetadataCache.getTopicMetadata( + streams.asScala.toSet, + systemName, + getTopicMetadata, + metadataTTL) + val result = metadata.map { + case (topic, topicMetadata) => { + KafkaUtil.maybeThrowException(topicMetadata.error.exception()) + val partitionsMap = topicMetadata.partitionsMetadata.map { + pm => + new Partition(pm.partitionId) -> new SystemStreamPartitionMetadata("", "", "") + }.toMap[Partition, SystemStreamPartitionMetadata] + (topic -> new SystemStreamMetadata(topic, partitionsMap.asJava)) + } + } + loop.done + result.asJava + }, + + (exception, loop) => { + warn("Unable to fetch last offsets for streams %s due to %s. Retrying." format (streams, exception)) + debug("Exception detail:", exception) + if (metadataTTL == Long.MaxValue) { + metadataTTL = 5000 // Revert to the default cache expiration + } + } + ).getOrElse(throw new SamzaException("Failed to get system stream metadata")) + } + + /** + * Returns the offset for the message after the specified offset for each + * SystemStreamPartition that was passed in. + */ + + override def getOffsetsAfter(offsets: java.util.Map[SystemStreamPartition, String]) = { + // This is safe to do with Kafka, even if a topic is key-deduped. If the + // offset doesn't exist on a compacted topic, Kafka will return the first + // message AFTER the offset that was specified in the fetch request. + offsets.asScala.mapValues(offset => (offset.toLong + 1).toString).asJava + } + + override def getSystemStreamMetadata(streams: java.util.Set[String]) = + getSystemStreamMetadata(streams, new ExponentialSleepStrategy(initialDelayMs = 500)).asJava + + /** + * Given a set of stream names (topics), fetch metadata from Kafka for each + * stream, and return a map from stream name to SystemStreamMetadata for + * each stream. This method will return null for oldest and newest offsets + * if a given SystemStreamPartition is empty. This method will block and + * retry indefinitely until it gets a successful response from Kafka. + */ + def getSystemStreamMetadata(streams: java.util.Set[String], retryBackoff: ExponentialSleepStrategy) = { + debug("Fetching system stream metadata for: %s" format streams) + var metadataTTL = Long.MaxValue // Trust the cache until we get an exception + retryBackoff.run( + loop => { + val metadata = TopicMetadataCache.getTopicMetadata( + streams.asScala.toSet, + systemName, + getTopicMetadata, + metadataTTL) + + debug("Got metadata for streams: %s" format metadata) + + val brokersToTopicPartitions = getTopicsAndPartitionsByBroker(metadata) + var oldestOffsets = Map[SystemStreamPartition, String]() + var newestOffsets = Map[SystemStreamPartition, String]() + var upcomingOffsets = Map[SystemStreamPartition, String]() + + // Get oldest, newest, and upcoming offsets for each topic and partition. + for ((broker, topicsAndPartitions) <- brokersToTopicPartitions) { + debug("Fetching offsets for %s:%s: %s" format (broker.host, broker.port, topicsAndPartitions)) + + val consumer = new SimpleConsumer(broker.host, broker.port, timeout, bufferSize, clientId) + try { + upcomingOffsets ++= getOffsets(consumer, topicsAndPartitions, OffsetRequest.LatestTime) + oldestOffsets ++= getOffsets(consumer, topicsAndPartitions, OffsetRequest.EarliestTime) + + // Kafka's "latest" offset is always last message in stream's offset + + // 1, so get newest message in stream by subtracting one. this is safe + // even for key-deduplicated streams, since the last message will + // never be deduplicated. + newestOffsets = upcomingOffsets.mapValues(offset => (offset.toLong - 1).toString) + // Keep only oldest/newest offsets where there is a message. Should + // return null offsets for empty streams. + upcomingOffsets.foreach { + case (topicAndPartition, offset) => + if (offset.toLong <= 0) { + debug("Stripping newest offsets for %s because the topic appears empty." format topicAndPartition) + newestOffsets -= topicAndPartition + debug("Setting oldest offset to 0 to consume from beginning") + oldestOffsets += (topicAndPartition -> "0") + } + } + } finally { + consumer.close + } + } + + val result = assembleMetadata(oldestOffsets, newestOffsets, upcomingOffsets) + loop.done + result + }, + + (exception, loop) => { + warn("Unable to fetch last offsets for streams %s due to %s. Retrying." format (streams, exception)) + debug("Exception detail:", exception) + metadataTTL = 5000 // Revert to the default cache expiration + }).getOrElse(throw new SamzaException("Failed to get system stream metadata")) + } + + /** + * Returns the newest offset for the specified SSP. + * This method is fast and targeted. It minimizes the number of kafka requests. + * It does not retry indefinitely if there is any failure. + * It returns null if the topic is empty. To get the offsets for *all* + * partitions, it would be more efficient to call getSystemStreamMetadata + */ + override def getNewestOffset(ssp: SystemStreamPartition, maxRetries: Integer) = { + debug("Fetching newest offset for: %s" format ssp) + var offset: String = null + var metadataTTL = Long.MaxValue // Trust the cache until we get an exception + var retries = maxRetries + new ExponentialSleepStrategy().run( + loop => { + val metadata = TopicMetadataCache.getTopicMetadata( + Set(ssp.getStream), + systemName, + getTopicMetadata, + metadataTTL) + debug("Got metadata for streams: %s" format metadata) + + val brokersToTopicPartitions = getTopicsAndPartitionsByBroker(metadata) + val topicAndPartition = new TopicAndPartition(ssp.getStream, ssp.getPartition.getPartitionId) + val broker = brokersToTopicPartitions.filter((e) => e._2.contains(topicAndPartition)).head._1 + + // Get oldest, newest, and upcoming offsets for each topic and partition. + debug("Fetching offset for %s:%s: %s" format (broker.host, broker.port, topicAndPartition)) + val consumer = new SimpleConsumer(broker.host, broker.port, timeout, bufferSize, clientId) + try { + offset = getOffsets(consumer, Set(topicAndPartition), OffsetRequest.LatestTime).head._2 + + // Kafka's "latest" offset is always last message in stream's offset + + // 1, so get newest message in stream by subtracting one. this is safe + // even for key-deduplicated streams, since the last message will + // never be deduplicated. + if (offset.toLong <= 0) { + debug("Stripping newest offsets for %s because the topic appears empty." format topicAndPartition) + offset = null + } else { + offset = (offset.toLong - 1).toString + } + } finally { + consumer.close + } + + debug("Got offset %s for %s." format(offset, ssp)) + loop.done + }, + + (exception, loop) => { + if (retries > 0) { + warn("Exception while trying to get offset for %s: %s. Retrying." format(ssp, exception)) + metadataTTL = 0L // Force metadata refresh + retries -= 1 + } else { + warn("Exception while trying to get offset for %s" format(ssp), exception) + loop.done + throw exception + } + }) + + offset + } + + /** + * Helper method to use topic metadata cache when fetching metadata, so we + * don't hammer Kafka more than we need to. + */ + def getTopicMetadata(topics: Set[String]) = { + new ClientUtilTopicMetadataStore(brokerListString, clientId, timeout) + .getTopicInfo(topics) + } + + /** + * Break topic metadata topic/partitions into per-broker map so that we can + * execute only one offset request per broker. + */ + private def getTopicsAndPartitionsByBroker(metadata: Map[String, TopicMetadata]) = { + val brokersToTopicPartitions = metadata + .values + // Convert the topic metadata to a Seq[(Broker, TopicAndPartition)] + .flatMap(topicMetadata => { + KafkaUtil.maybeThrowException(topicMetadata.error.exception()) + topicMetadata + .partitionsMetadata + // Convert Seq[PartitionMetadata] to Seq[(Broker, TopicAndPartition)] + .map(partitionMetadata => { + val topicAndPartition = new TopicAndPartition(topicMetadata.topic, partitionMetadata.partitionId) + val leader = partitionMetadata + .leader + .getOrElse(throw new SamzaException("Need leaders for all partitions when fetching offsets. No leader available for TopicAndPartition: %s" format topicAndPartition)) + (leader, topicAndPartition) + }) + }) + + // Convert to a Map[Broker, Seq[(Broker, TopicAndPartition)]] + .groupBy(_._1) + // Convert to a Map[Broker, Set[TopicAndPartition]] + .mapValues(_.map(_._2).toSet) + + debug("Got topic partition data for brokers: %s" format brokersToTopicPartitions) + + brokersToTopicPartitions + } + + /** + * Use a SimpleConsumer to fetch either the earliest or latest offset from + * Kafka for each topic/partition in the topicsAndPartitions set. It is + * assumed that all topics/partitions supplied reside on the broker that the + * consumer is connected to. + */ + private def getOffsets(consumer: SimpleConsumer, topicsAndPartitions: Set[TopicAndPartition], earliestOrLatest: Long) = { + debug("Getting offsets for %s using earliest/latest value of %s." format (topicsAndPartitions, earliestOrLatest)) + + var offsets = Map[SystemStreamPartition, String]() + val partitionOffsetInfo = topicsAndPartitions + .map(topicAndPartition => (topicAndPartition, PartitionOffsetRequestInfo(earliestOrLatest, 1))) + .toMap + val brokerOffsets = consumer + .getOffsetsBefore(new OffsetRequest(partitionOffsetInfo)) + .partitionErrorAndOffsets + .mapValues(partitionErrorAndOffset => { + KafkaUtil.maybeThrowException(partitionErrorAndOffset.error.exception()) + partitionErrorAndOffset.offsets.head + }) + + for ((topicAndPartition, offset) <- brokerOffsets) { + offsets += new SystemStreamPartition(systemName, topicAndPartition.topic, new Partition(topicAndPartition.partition)) -> offset.toString + } + + debug("Got offsets for %s using earliest/latest value of %s: %s" format (topicsAndPartitions, earliestOrLatest, offsets)) + + offsets + } + + /** + * @inheritdoc + */ + override def createStream(spec: StreamSpec): Boolean = { + info("Create topic %s in system %s" format (spec.getPhysicalName, systemName)) + val kSpec = toKafkaSpec(spec) + var streamCreated = false + + new ExponentialSleepStrategy(initialDelayMs = 500).run( + loop => { + val zkClient = connectZk() + try { + AdminUtils.createTopic( + zkClient, + kSpec.getPhysicalName, + kSpec.getPartitionCount, + kSpec.getReplicationFactor, + kSpec.getProperties) + } finally { + zkClient.close + } + + streamCreated = true + loop.done + }, + + (exception, loop) => { + exception match { + case e: TopicExistsException => + streamCreated = false + loop.done + case e: Exception => + warn("Failed to create topic %s: %s. Retrying." format (spec.getPhysicalName, e)) + debug("Exception detail:", e) + } + }) + + streamCreated + } + + /** + * Converts a StreamSpec into a KafakStreamSpec. Special handling for coordinator and changelog stream. + * @param spec a StreamSpec object + * @return KafkaStreamSpec object + */ + def toKafkaSpec(spec: StreamSpec): KafkaStreamSpec = { + if (spec.isChangeLogStream) { + val topicName = spec.getPhysicalName + val topicMeta = topicMetaInformation.getOrElse(topicName, throw new StreamValidationException("Unable to find topic information for topic " + topicName)) + new KafkaStreamSpec(spec.getId, topicName, systemName, spec.getPartitionCount, topicMeta.replicationFactor, + topicMeta.kafkaProps) + } else if (spec.isCoordinatorStream){ + new KafkaStreamSpec(spec.getId, spec.getPhysicalName, systemName, 1, coordinatorStreamReplicationFactor, + coordinatorStreamProperties) + } else if (intermediateStreamProperties.contains(spec.getId)) { + KafkaStreamSpec.fromSpec(spec).copyWithProperties(intermediateStreamProperties(spec.getId)) + } else { + KafkaStreamSpec.fromSpec(spec) + } + } + + /** + * @inheritdoc + * + * Validates a stream in Kafka. Should not be called before createStream(), + * since ClientUtils.fetchTopicMetadata(), used by different Kafka clients, + * is not read-only and will auto-create a new topic. + */ + override def validateStream(spec: StreamSpec): Unit = { + val topicName = spec.getPhysicalName + info("Validating topic %s." format topicName) + + val retryBackoff: ExponentialSleepStrategy = new ExponentialSleepStrategy + var metadataTTL = Long.MaxValue // Trust the cache until we get an exception + retryBackoff.run( + loop => { + val metadataStore = new ClientUtilTopicMetadataStore(brokerListString, clientId, timeout) + val topicMetadataMap = TopicMetadataCache.getTopicMetadata(Set(topicName), systemName, metadataStore.getTopicInfo, metadataTTL) + val topicMetadata = topicMetadataMap(topicName) + KafkaUtil.maybeThrowException(topicMetadata.error.exception()) + + val partitionCount = topicMetadata.partitionsMetadata.length + if (partitionCount != spec.getPartitionCount) { + throw new StreamValidationException("Topic validation failed for topic %s because partition count %s did not match expected partition count of %d" format (topicName, topicMetadata.partitionsMetadata.length, spec.getPartitionCount)) + } + + info("Successfully validated topic %s." format topicName) + loop.done + }, + + (exception, loop) => { + exception match { + case e: StreamValidationException => throw e + case e: Exception => + warn("While trying to validate topic %s: %s. Retrying." format (topicName, e)) + debug("Exception detail:", e) + metadataTTL = 5000L // Revert to the default value + } + }) + } + + /** + * @inheritdoc + * + * Delete a stream in Kafka. Deleting topics works only when the broker is configured with "delete.topic.enable=true". + * Otherwise it's a no-op. + */ + override def clearStream(spec: StreamSpec): Boolean = { + info("Delete topic %s in system %s" format (spec.getPhysicalName, systemName)) + val kSpec = KafkaStreamSpec.fromSpec(spec) + var retries = CLEAR_STREAM_RETRIES + new ExponentialSleepStrategy().run( + loop => { + val zkClient = connectZk() + try { + AdminUtils.deleteTopic( + zkClient, + kSpec.getPhysicalName) + } finally { + zkClient.close + } + + loop.done + }, + + (exception, loop) => { + if (retries > 0) { + warn("Exception while trying to delete topic %s: %s. Retrying." format (spec.getPhysicalName, exception)) + retries -= 1 + } else { + warn("Fail to delete topic %s: %s" format (spec.getPhysicalName, exception)) + loop.done + throw exception + } + }) + + val topicMetadata = getTopicMetadata(Set(kSpec.getPhysicalName)).get(kSpec.getPhysicalName).get + topicMetadata.partitionsMetadata.isEmpty + } + + /** + * @inheritdoc + * + * Delete records up to (and including) the provided ssp offsets for all system stream partitions specified in the map + * This only works with Kafka cluster 0.11 or later. Otherwise it's a no-op. + */ + override def deleteMessages(offsets: util.Map[SystemStreamPartition, String]) { + if (!running) { + throw new SamzaException(s"KafkaSystemAdmin has not started yet for system $systemName") + } + if (deleteCommittedMessages) { + val nextOffsets = offsets.asScala.toSeq.map { case (systemStreamPartition, offset) => + (new TopicPartition(systemStreamPartition.getStream, systemStreamPartition.getPartition.getPartitionId), offset.toLong + 1) + }.toMap + adminClient.deleteRecordsBefore(nextOffsets) + deleteMessagesCalled = true + } + } + + /** + * Compare the two offsets. Returns x where x < 0 if offset1 < offset2; + * x == 0 if offset1 == offset2; x > 0 if offset1 > offset2. + * + * Currently it's used in the context of the broadcast streams to detect + * the mismatch between two streams when consuming the broadcast streams. + */ + override def offsetComparator(offset1: String, offset2: String): Integer = { + offset1.toLong compare offset2.toLong + } +} diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.java b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.java new file mode 100644 index 0000000000..6a7200c6cf --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.java @@ -0,0 +1,371 @@ + +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.samza.system.kafka_deprecated; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import kafka.common.TopicAndPartition; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; +import org.apache.samza.SamzaException; +import org.apache.samza.config.Config; +import org.apache.samza.config.KafkaConfig; +import org.apache.samza.config.KafkaConsumerConfig; +import org.apache.samza.system.IncomingMessageEnvelope; +import org.apache.samza.system.SystemConsumer; +import org.apache.samza.system.SystemStreamPartition; +import org.apache.samza.util.BlockingEnvelopeMap; +import org.apache.samza.util.Clock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Option; + + +public class KafkaSystemConsumer extends BlockingEnvelopeMap implements SystemConsumer { + + private static final Logger LOG = LoggerFactory.getLogger(KafkaSystemConsumer.class); + + private static final long FETCH_THRESHOLD = 50000; + private static final long FETCH_THRESHOLD_BYTES = -1L; + + private final Consumer kafkaConsumer; + private final String systemName; + private final String clientId; + private final AtomicBoolean stopped = new AtomicBoolean(false); + private final AtomicBoolean started = new AtomicBoolean(false); + private final Config config; + private final boolean fetchThresholdBytesEnabled; + private final KafkaSystemConsumerMetrics metrics; + + // This sink is used to transfer the messages from the proxy/consumer to the BlockingEnvelopeMap. + final KafkaConsumerMessageSink messageSink; + + // This proxy contains a separate thread, which reads kafka messages (with consumer.poll()) and populates + // BlockingEnvelopMap's buffers. + final private KafkaConsumerProxy proxy; + + // keep registration data until the start - mapping between registered SSPs and topicPartitions, and their offsets + final Map topicPartitionsToOffset = new HashMap<>(); + final Map topicPartitionsToSSP = new HashMap<>(); + + long perPartitionFetchThreshold; + long perPartitionFetchThresholdBytes; + + /** + * Create a KafkaSystemConsumer for the provided {@code systemName} + * @param systemName system name for which we create the consumer + * @param config application config + * @param metrics metrics for this KafkaSystemConsumer + * @param clock system clock + */ + public KafkaSystemConsumer(Consumer kafkaConsumer, String systemName, Config config, String clientId, + KafkaSystemConsumerMetrics metrics, Clock clock) { + + super(metrics.registry(), clock, metrics.getClass().getName()); + + this.kafkaConsumer = kafkaConsumer; + this.clientId = clientId; + this.systemName = systemName; + this.config = config; + this.metrics = metrics; + + fetchThresholdBytesEnabled = new KafkaConfig(config).isConsumerFetchThresholdBytesEnabled(systemName); + + // create a sink for passing the messages between the proxy and the consumer + messageSink = new KafkaConsumerMessageSink(); + + // Create the proxy to do the actual message reading. + String metricName = String.format("%s", systemName); + proxy = new KafkaConsumerProxy(kafkaConsumer, systemName, clientId, messageSink, metrics, metricName); + LOG.info("{}: Created KafkaConsumerProxy {} ", this, proxy); + } + + /** + * Create internal kafka consumer object, which will be used in the Proxy. + * @param systemName system name for which we create the consumer + * @param clientId client id to use int the kafka client + * @param config config + * @return kafka consumer object + */ + public static KafkaConsumer getKafkaConsumerImpl(String systemName, String clientId, Config config) { + + // extract kafka client configs + KafkaConsumerConfig consumerConfig = KafkaConsumerConfig.getKafkaSystemConsumerConfig(config, systemName, clientId); + + LOG.info("{}: KafkaClient properties {}", systemName, consumerConfig); + + return new KafkaConsumer(consumerConfig); + } + + @Override + public void start() { + if (!started.compareAndSet(false, true)) { + LOG.warn("{}: Attempting to start the consumer for the second (or more) time.", this); + return; + } + if (stopped.get()) { + LOG.error("{}: Attempting to start a stopped consumer", this); + return; + } + // initialize the subscriptions for all the registered TopicPartitions + startSubscription(); + // needs to be called after all the registrations are completed + setFetchThresholds(); + + startConsumer(); + LOG.info("{}: Consumer started", this); + } + + private void startSubscription() { + //subscribe to all the registered TopicPartitions + LOG.info("{}: Consumer subscribes to {}", this, topicPartitionsToSSP.keySet()); + try { + synchronized (kafkaConsumer) { + // we are using assign (and not subscribe), so we need to specify both topic and partition + kafkaConsumer.assign(topicPartitionsToSSP.keySet()); + } + } catch (Exception e) { + throw new SamzaException("Consumer subscription failed for " + this, e); + } + } + + /** + * Set the offsets to start from. + * Register the TopicPartitions with the proxy. + * Start the proxy. + */ + void startConsumer() { + // set the offset for each TopicPartition + if (topicPartitionsToOffset.size() <= 0) { + LOG.error ("{}: Consumer is not subscribed to any SSPs", this); + } + + topicPartitionsToOffset.forEach((tp, startingOffsetString) -> { + long startingOffset = Long.valueOf(startingOffsetString); + + try { + synchronized (kafkaConsumer) { + kafkaConsumer.seek(tp, startingOffset); // this value should already be the 'upcoming' value + } + } catch (Exception e) { + // all recoverable execptions are handled by the client. + // if we get here there is nothing left to do but bail out. + String msg = + String.format("%s: Got Exception while seeking to %s for partition %s", this, startingOffsetString, tp); + LOG.error(msg, e); + throw new SamzaException(msg, e); + } + + LOG.info("{}: Changing consumer's starting offset for tp = %s to %s", this, tp, startingOffsetString); + + // add the partition to the proxy + proxy.addTopicPartition(topicPartitionsToSSP.get(tp), startingOffset); + }); + + // start the proxy thread + if (proxy != null && !proxy.isRunning()) { + LOG.info("{}: Starting proxy {}", this, proxy); + proxy.start(); + } + } + + private void setFetchThresholds() { + // get the thresholds, and set defaults if not defined. + KafkaConfig kafkaConfig = new KafkaConfig(config); + + Option fetchThresholdOption = kafkaConfig.getConsumerFetchThreshold(systemName); + long fetchThreshold = FETCH_THRESHOLD; + if (fetchThresholdOption.isDefined()) { + fetchThreshold = Long.valueOf(fetchThresholdOption.get()); + } + + Option fetchThresholdBytesOption = kafkaConfig.getConsumerFetchThresholdBytes(systemName); + long fetchThresholdBytes = FETCH_THRESHOLD_BYTES; + if (fetchThresholdBytesOption.isDefined()) { + fetchThresholdBytes = Long.valueOf(fetchThresholdBytesOption.get()); + } + + int numPartitions = topicPartitionsToSSP.size(); + if (numPartitions != topicPartitionsToOffset.size()) { + throw new SamzaException("topicPartitionsToSSP.size() doesn't match topicPartitionsToOffset.size()"); + } + + + if (numPartitions > 0) { + perPartitionFetchThreshold = fetchThreshold / numPartitions; + if (fetchThresholdBytesEnabled) { + // currently this feature cannot be enabled, because we do not have the size of the messages available. + // messages get double buffered, hence divide by 2 + perPartitionFetchThresholdBytes = (fetchThresholdBytes / 2) / numPartitions; + } + } + LOG.info("{}: fetchThresholdBytes = {}; fetchThreshold={}; numPartitions={}, perPartitionFetchThreshold={}, perPartitionFetchThresholdBytes(0 if disabled)={}", + this, fetchThresholdBytes, fetchThreshold, numPartitions, perPartitionFetchThreshold, perPartitionFetchThresholdBytes); + } + + @Override + public void stop() { + if (!stopped.compareAndSet(false, true)) { + LOG.warn("{}: Attempting to stop stopped consumer.", this); + return; + } + + LOG.info("{}: Stopping Samza kafkaConsumer ", this); + + // stop the proxy (with 1 minute timeout) + if (proxy != null) { + LOG.info("{}: Stopping proxy {}", this, proxy); + proxy.stop(TimeUnit.SECONDS.toMillis(60)); + } + + try { + synchronized (kafkaConsumer) { + LOG.info("{}: Closing kafkaSystemConsumer {}", this, kafkaConsumer); + kafkaConsumer.close(); + } + } catch (Exception e) { + LOG.warn("{}: Failed to stop KafkaSystemConsumer.", this, e); + } + } + + /** + * record the ssp and the offset. Do not submit it to the consumer yet. + * @param systemStreamPartition ssp to register + * @param offset offset to register with + */ + @Override + public void register(SystemStreamPartition systemStreamPartition, String offset) { + if (started.get()) { + String msg = String.format("%s: Trying to register partition after consumer has been started. ssp=%s", this, + systemStreamPartition); + throw new SamzaException(msg); + } + + if (!systemStreamPartition.getSystem().equals(systemName)) { + LOG.warn("{}: ignoring SSP {}, because this consumer's system doesn't match.", this, systemStreamPartition); + return; + } + LOG.info("{}: Registering ssp = {} with offset {}", this, systemStreamPartition, offset); + + super.register(systemStreamPartition, offset); + + TopicPartition tp = toTopicPartition(systemStreamPartition); + + topicPartitionsToSSP.put(tp, systemStreamPartition); + + String existingOffset = topicPartitionsToOffset.get(tp); + // register the older (of the two) offset in the consumer, to guarantee we do not miss any messages. + if (existingOffset == null || compareOffsets(existingOffset, offset) > 0) { + topicPartitionsToOffset.put(tp, offset); + } + + metrics.registerTopicAndPartition(toTopicAndPartition(tp)); + } + + /** + * Compare two String offsets. + * Note. There is a method in KafkaSystemAdmin that does that, but that would require instantiation of systemadmin for each consumer. + * @return see {@link Long#compareTo(Long)} + */ + private static int compareOffsets(String offset1, String offset2) { + return Long.valueOf(offset1).compareTo(Long.valueOf(offset2)); + } + + @Override + public String toString() { + return String.format("%s:%s", systemName, clientId); + } + + @Override + public Map> poll( + Set systemStreamPartitions, long timeout) throws InterruptedException { + + // check if the proxy is running + if (!proxy.isRunning()) { + stop(); + String message = String.format("%s: KafkaConsumerProxy has stopped.", this); + throw new SamzaException(message, proxy.getFailureCause()); + } + + return super.poll(systemStreamPartitions, timeout); + } + + /** + * convert from TopicPartition to TopicAndPartition + */ + public static TopicAndPartition toTopicAndPartition(TopicPartition tp) { + return new TopicAndPartition(tp.topic(), tp.partition()); + } + + /** + * convert to TopicPartition from SystemStreamPartition + */ + public static TopicPartition toTopicPartition(SystemStreamPartition ssp) { + return new TopicPartition(ssp.getStream(), ssp.getPartition().getPartitionId()); + } + + /** + * return system name for this consumer + * @return system name + */ + public String getSystemName() { + return systemName; + } + + public class KafkaConsumerMessageSink { + + public void setIsAtHighWatermark(SystemStreamPartition ssp, boolean isAtHighWatermark) { + setIsAtHead(ssp, isAtHighWatermark); + } + + boolean needsMoreMessages(SystemStreamPartition ssp) { + LOG.debug("{}: needsMoreMessages from following SSP: {}. fetchLimitByBytes enabled={}; messagesSizeInQueue={};" + + "(limit={}); messagesNumInQueue={}(limit={};", this, ssp, fetchThresholdBytesEnabled, + getMessagesSizeInQueue(ssp), perPartitionFetchThresholdBytes, getNumMessagesInQueue(ssp), + perPartitionFetchThreshold); + + if (fetchThresholdBytesEnabled) { + return getMessagesSizeInQueue(ssp) < perPartitionFetchThresholdBytes; + } else { + return getNumMessagesInQueue(ssp) < perPartitionFetchThreshold; + } + } + + void addMessage(SystemStreamPartition ssp, IncomingMessageEnvelope envelope) { + LOG.trace("{}: Incoming message ssp = {}: envelope = {}.", this, ssp, envelope); + + try { + put(ssp, envelope); + } catch (InterruptedException e) { + throw new SamzaException( + String.format("%s: Consumer was interrupted while trying to add message with offset %s for ssp %s", this, + envelope.getOffset(), ssp)); + } + } + } +} diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumerMetrics.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumerMetrics.scala new file mode 100644 index 0000000000..b01fecc1b9 --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumerMetrics.scala @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.system.kafka_deprecated + +import java.util.concurrent.ConcurrentHashMap + +import kafka.common.TopicAndPartition +import org.apache.samza.metrics._ + +class KafkaSystemConsumerMetrics(val systemName: String = "unknown", val registry: MetricsRegistry = new MetricsRegistryMap) extends MetricsHelper { + val offsets = new ConcurrentHashMap[TopicAndPartition, Counter] + val bytesRead = new ConcurrentHashMap[TopicAndPartition, Counter] + val reads = new ConcurrentHashMap[TopicAndPartition, Counter] + val lag = new ConcurrentHashMap[TopicAndPartition, Gauge[Long]] + val highWatermark = new ConcurrentHashMap[TopicAndPartition, Gauge[Long]] + + val clientBytesRead = new ConcurrentHashMap[String, Counter] + val clientReads = new ConcurrentHashMap[String, Counter] + val clientSkippedFetchRequests = new ConcurrentHashMap[String, Counter] + val topicPartitions = new ConcurrentHashMap[String, Gauge[Int]] + + def registerTopicAndPartition(tp: TopicAndPartition) = { + if (!offsets.contains(tp)) { + offsets.put(tp, newCounter("%s-%s-offset-change" format(tp.topic, tp.partition))) + bytesRead.put(tp, newCounter("%s-%s-bytes-read" format(tp.topic, tp.partition))) + reads.put(tp, newCounter("%s-%s-messages-read" format(tp.topic, tp.partition))) + highWatermark.put(tp, newGauge("%s-%s-high-watermark" format(tp.topic, tp.partition), -1L)) + lag.put(tp, newGauge("%s-%s-messages-behind-high-watermark" format(tp.topic, tp.partition), 0L)) + } + } + + def registerClientProxy(clientName: String) { + clientBytesRead.put(clientName, newCounter("%s-bytes-read" format clientName)) + clientReads.put((clientName), newCounter("%s-messages-read" format clientName)) + clientSkippedFetchRequests.put((clientName), newCounter("%s-skipped-fetch-requests" format clientName)) + topicPartitions.put(clientName, newGauge("%s-registered-topic-partitions" format clientName, 0)) + } + + // java friendlier interfaces + // Gauges + def setNumTopicPartitions(clientName: String, value: Int) { + topicPartitions.get(clientName).set(value) + } + + def setLagValue(topicAndPartition: TopicAndPartition, value: Long) { + lag.get((topicAndPartition)).set(value); + } + + def setHighWatermarkValue(topicAndPartition: TopicAndPartition, value: Long) { + highWatermark.get((topicAndPartition)).set(value); + } + + // Counters + def incClientReads(clientName: String) { + clientReads.get(clientName).inc + } + + def incReads(topicAndPartition: TopicAndPartition) { + reads.get(topicAndPartition).inc; + } + + def incBytesReads(topicAndPartition: TopicAndPartition, inc: Long) { + bytesRead.get(topicAndPartition).inc(inc); + } + + def incClientBytesReads(clientName: String, incBytes: Long) { + clientBytesRead.get(clientName).inc(incBytes) + } + + def incClientSkippedFetchRequests(clientName: String) { + clientSkippedFetchRequests.get(clientName).inc() + } + + def setOffsets(topicAndPartition: TopicAndPartition, offset: Long) { + offsets.get(topicAndPartition).set(offset) + } + + override def getPrefix = systemName + "-" +} diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala new file mode 100644 index 0000000000..933dd59105 --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.system.kafka_deprecated +import java.util.Properties +import kafka.utils.ZkUtils +import org.apache.samza.SamzaException +import org.apache.samza.config.ApplicationConfig.ApplicationMode +import org.apache.samza.util.{Logging, KafkaUtil, ExponentialSleepStrategy, ClientUtilTopicMetadataStore} +import org.apache.samza.config.{KafkaConfig, ApplicationConfig, StreamConfig, Config} +import org.apache.samza.metrics.MetricsRegistry +import org.apache.samza.config.KafkaConfig.Config2Kafka +import org.apache.samza.config.TaskConfig.Config2Task +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.samza.system.SystemFactory +import org.apache.samza.config.StorageConfig._ +import org.apache.samza.system.SystemProducer +import org.apache.samza.system.SystemAdmin +import org.apache.samza.config.SystemConfig.Config2System +import org.apache.samza.system.SystemConsumer + +object KafkaSystemFactory extends Logging { + def getInjectedProducerProperties(systemName: String, config: Config) = if (config.isChangelogSystem(systemName)) { + warn("System name '%s' is being used as a changelog. Disabling compression since Kafka does not support compression for log compacted topics." format systemName) + Map[String, String]("compression.type" -> "none") + } else { + Map[String, String]() + } +} + +class KafkaSystemFactory extends SystemFactory with Logging { + def getConsumer(systemName: String, config: Config, registry: MetricsRegistry): SystemConsumer = { + val clientId = KafkaUtil.getClientId("samza-consumer", config) + val metrics = new KafkaSystemConsumerMetrics(systemName, registry) + + // Kind of goofy to need a producer config for consumers, but we need metadata. + val producerConfig = config.getKafkaSystemProducerConfig(systemName, clientId) + val bootstrapServers = producerConfig.bootsrapServers + val consumerConfig = config.getKafkaSystemConsumerConfig(systemName, clientId) + + val timeout = consumerConfig.socketTimeoutMs + val bufferSize = consumerConfig.socketReceiveBufferBytes + val fetchSize = new StreamFetchSizes(consumerConfig.fetchMessageMaxBytes, config.getFetchMessageMaxBytesTopics(systemName)) + val consumerMinSize = consumerConfig.fetchMinBytes + val consumerMaxWait = consumerConfig.fetchWaitMaxMs + val autoOffsetResetDefault = consumerConfig.autoOffsetReset + val autoOffsetResetTopics = config.getAutoOffsetResetTopics(systemName) + val fetchThreshold = config.getConsumerFetchThreshold(systemName).getOrElse("50000").toInt + val fetchThresholdBytes = config.getConsumerFetchThresholdBytes(systemName).getOrElse("-1").toLong + val offsetGetter = new GetOffset(autoOffsetResetDefault, autoOffsetResetTopics) + val metadataStore = new ClientUtilTopicMetadataStore(bootstrapServers, clientId, timeout) + + new KafkaSystemConsumer( + systemName = systemName, + systemAdmin = getAdmin(systemName, config), + metrics = metrics, + metadataStore = metadataStore, + clientId = clientId, + timeout = timeout, + bufferSize = bufferSize, + fetchSize = fetchSize, + consumerMinSize = consumerMinSize, + consumerMaxWait = consumerMaxWait, + fetchThreshold = fetchThreshold, + fetchThresholdBytes = fetchThresholdBytes, + fetchLimitByBytesEnabled = config.isConsumerFetchThresholdBytesEnabled(systemName), + offsetGetter = offsetGetter) + } + + def getProducer(systemName: String, config: Config, registry: MetricsRegistry): SystemProducer = { + val clientId = KafkaUtil.getClientId("samza-producer", config) + val injectedProps = KafkaSystemFactory.getInjectedProducerProperties(systemName, config) + val producerConfig = config.getKafkaSystemProducerConfig(systemName, clientId, injectedProps) + val getProducer = () => { new KafkaProducer[Array[Byte], Array[Byte]](producerConfig.getProducerProperties) } + val metrics = new KafkaSystemProducerMetrics(systemName, registry) + + // Unlike consumer, no need to use encoders here, since they come for free + // inside the producer configs. Kafka's producer will handle all of this + // for us. + + new KafkaSystemProducer( + systemName, + new ExponentialSleepStrategy(initialDelayMs = producerConfig.reconnectIntervalMs), + getProducer, + metrics, + dropProducerExceptions = config.getDropProducerError) + } + + def getAdmin(systemName: String, config: Config): SystemAdmin = { + val clientId = KafkaUtil.getClientId("samza-admin", config) + val producerConfig = config.getKafkaSystemProducerConfig(systemName, clientId) + val bootstrapServers = producerConfig.bootsrapServers + val consumerConfig = config.getKafkaSystemConsumerConfig(systemName, clientId) + val timeout = consumerConfig.socketTimeoutMs + val bufferSize = consumerConfig.socketReceiveBufferBytes + val zkConnect = Option(consumerConfig.zkConnect) + .getOrElse(throw new SamzaException("no zookeeper.connect defined in config")) + val connectZk = () => { + ZkUtils(zkConnect, 6000, 6000, false) + } + val coordinatorStreamProperties = getCoordinatorTopicProperties(config) + val coordinatorStreamReplicationFactor = config.getCoordinatorReplicationFactor.toInt + val storeToChangelog = config.getKafkaChangelogEnabledStores() + // Construct the meta information for each topic, if the replication factor is not defined, we use 2 as the number of replicas for the change log stream. + val topicMetaInformation = storeToChangelog.map{case (storeName, topicName) => + { + val replicationFactor = config.getChangelogStreamReplicationFactor(storeName).toInt + val changelogInfo = ChangelogInfo(replicationFactor, config.getChangelogKafkaProperties(storeName)) + info("Creating topic meta information for topic: %s with replication factor: %s" format (topicName, replicationFactor)) + (topicName, changelogInfo) + }} + + val deleteCommittedMessages = config.deleteCommittedMessages(systemName).exists(isEnabled => isEnabled.toBoolean) + val intermediateStreamProperties: Map[String, Properties] = getIntermediateStreamProperties(config) + new KafkaSystemAdmin( + systemName, + bootstrapServers, + connectZk, + coordinatorStreamProperties, + coordinatorStreamReplicationFactor, + timeout, + bufferSize, + clientId, + topicMetaInformation, + intermediateStreamProperties, + deleteCommittedMessages) + } + + def getCoordinatorTopicProperties(config: Config) = { + val segmentBytes = config.getCoordinatorSegmentBytes + (new Properties /: Map( + "cleanup.policy" -> "compact", + "segment.bytes" -> segmentBytes)) { case (props, (k, v)) => props.put(k, v); props } + } + + def getIntermediateStreamProperties(config : Config): Map[String, Properties] = { + val appConfig = new ApplicationConfig(config) + if (appConfig.getAppMode == ApplicationMode.BATCH) { + val streamConfig = new StreamConfig(config) + streamConfig.getStreamIds().filter(streamConfig.getIsIntermediateStream(_)).map(streamId => { + val properties = new Properties() + properties.putAll(streamConfig.getStreamProperties(streamId)) + properties.putIfAbsent("retention.ms", String.valueOf(KafkaConfig.DEFAULT_RETENTION_MS_FOR_BATCH)) + (streamId, properties) + }).toMap + } else { + Map() + } + } +} diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemProducer.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemProducer.scala new file mode 100644 index 0000000000..af393785e8 --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemProducer.scala @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.system.kafka_deprecated + + +import java.util.concurrent.atomic.AtomicReference +import java.util.concurrent.TimeUnit + +import org.apache.kafka.clients.producer.Callback +import org.apache.kafka.clients.producer.Producer +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.clients.producer.RecordMetadata +import org.apache.kafka.common.PartitionInfo +import org.apache.samza.system.OutgoingMessageEnvelope +import org.apache.samza.system.SystemProducer +import org.apache.samza.system.SystemProducerException +import org.apache.samza.util.ExponentialSleepStrategy +import org.apache.samza.util.KafkaUtil +import org.apache.samza.util.Logging +import org.apache.samza.util.TimerUtil + +class KafkaSystemProducer(systemName: String, + retryBackoff: ExponentialSleepStrategy = new ExponentialSleepStrategy, + getProducer: () => Producer[Array[Byte], Array[Byte]], + metrics: KafkaSystemProducerMetrics, + val clock: () => Long = () => System.nanoTime, + val dropProducerExceptions: Boolean = false) extends SystemProducer with Logging with TimerUtil { + + // Represents a fatal error that caused the producer to close. + val fatalException: AtomicReference[SystemProducerException] = new AtomicReference[SystemProducerException]() + val producerRef: AtomicReference[Producer[Array[Byte], Array[Byte]]] = new AtomicReference[Producer[Array[Byte], Array[Byte]]]() + val producerCreationLock: Object = new Object + @volatile var stopped = false + + def start(): Unit = { + producerRef.set(getProducer()) + } + + def stop() { + info("Stopping producer for system: " + this.systemName) + + stopped = true + val currentProducer = producerRef.getAndSet(null) + try { + if (currentProducer != null) { + currentProducer.close // Also performs the equivalent of a flush() + } + + val exception = fatalException.get() + if (exception != null) { + error("Observed an earlier send() error while closing producer", exception) + } + } catch { + case e: Exception => error("Error while closing producer for system: " + systemName, e) + } + } + + def register(source: String) { + } + + def send(source: String, envelope: OutgoingMessageEnvelope) { + trace("Enqueuing message: %s, %s." format (source, envelope)) + + val topicName = envelope.getSystemStream.getStream + if (topicName == null || topicName.isEmpty) { + throw new IllegalArgumentException("Invalid system stream: " + envelope.getSystemStream) + } + + val globalProducerException = fatalException.get() + if (globalProducerException != null) { + metrics.sendFailed.inc + throw new SystemProducerException("Producer was unable to recover from previous exception.", globalProducerException) + } + + val currentProducer = getOrCreateCurrentProducer + + // Java-based Kafka producer API requires an "Integer" type partitionKey and does not allow custom overriding of Partitioners + // Any kind of custom partitioning has to be done on the client-side + val partitions: java.util.List[PartitionInfo] = currentProducer.partitionsFor(topicName) + val partitionKey = if (envelope.getPartitionKey != null) KafkaUtil.getIntegerPartitionKey(envelope, partitions) else null + val record = new ProducerRecord(envelope.getSystemStream.getStream, + partitionKey, + envelope.getKey.asInstanceOf[Array[Byte]], + envelope.getMessage.asInstanceOf[Array[Byte]]) + + try { + currentProducer.send(record, new Callback { + def onCompletion(metadata: RecordMetadata, exception: Exception): Unit = { + if (exception == null) { + metrics.sendSuccess.inc + } else { + val producerException = new SystemProducerException("Failed to send message for Source: %s on System:%s Topic:%s Partition:%s" + .format(source, systemName, topicName, partitionKey), exception) + + handleFatalSendException(currentProducer, producerException) + } + } + }) + metrics.sends.inc + } catch { + case originalException : Exception => + val producerException = new SystemProducerException("Failed to send message for Source: %s on System:%s Topic:%s Partition:%s" + .format(source, systemName, topicName, partitionKey), originalException) + + metrics.sendFailed.inc + error("Got a synchronous error from Kafka producer.", producerException) + // Synchronous exceptions are always recoverable so propagate it up and let the user decide + throw producerException + } + } + + def flush(source: String) { + updateTimer(metrics.flushNs) { + metrics.flushes.inc + + val currentProducer = producerRef.get() + if (currentProducer == null) { + if (dropProducerExceptions) { + // No producer to flush, but we're ignoring exceptions so just return. + warn("Skipping flush because the Kafka producer is null.") + metrics.flushFailed.inc + return + } + throw new SystemProducerException("Kafka producer is null.") + } + + // Flush only throws InterruptedException, all other errors are handled in send() callbacks + currentProducer.flush() + + // Invariant: At this point either + // 1. The producer is fine and there are no exceptions to handle OR + // 2. The producer is closed and one or more sources have exceptions to handle + // 2a. All new sends get a ProducerClosedException or IllegalStateException (depending on kafka version) + // 2b. There are no messages in flight because the producer is closed + + // We must check for an exception AFTER flush() because when flush() returns all callbacks for messages sent + // in that flush() are guaranteed to have completed and we update the exception in the callback. + // If there is an exception, we rethrow it here to prevent the checkpoint. + val exception = fatalException.get() + if (exception != null) { + metrics.flushFailed.inc + throw new SystemProducerException("Flush failed. One or more batches of messages were not sent!", exception) + } + trace("Flushed %s." format source) + } + } + + + /** + * Handles a fatal exception by closing the producer and either recreating it or storing the exception + * to rethrow later, depending on the value of dropProducerExceptions. + * + * @param currentProducer the current producer for which the exception occurred. Must not be null. + * @param producerException the exception to handle. + */ + private def handleFatalSendException(currentProducer: Producer[Array[Byte], Array[Byte]], producerException: SystemProducerException): Unit = { + metrics.sendFailed.inc + error(producerException) + // The SystemProducer API is synchronous, so there's no way for us to guarantee that an exception will + // be handled by the Task before we recreate the producer, and if it isn't handled, a concurrent send() from another + // Task could send on the new producer before the first Task properly handled the exception and produce out of order messages. + // So we have to handle it right here in the SystemProducer. + if (dropProducerExceptions) { + warn("Ignoring producer exception. All messages in the failed producer request will be dropped!") + + // Prevent each callback from closing and nulling producer for the same failure. + if (currentProducer == producerRef.get()) { + info("Closing producer for system %s." format systemName) + try { + // send()s can get ProducerClosedException if the producer is stopped after they get the currentProducer + // reference but before producer.send() returns. That's ONLY ok when dropProducerExceptions is true. + // Also, when producer.close(0) is invoked on the Kafka IO thread, when it returns there will be no more + // messages sent over the wire. This is key to ensuring no out-of-order messages as a result of recreating + // the producer. + currentProducer.close(0, TimeUnit.MILLISECONDS) + } catch { + case exception: Exception => error("Exception while closing producer.", exception) + } + producerRef.compareAndSet(currentProducer, null) + } + } else { + // If there is an exception in the callback, it means that the Kafka producer has exhausted the max-retries + // Close producer to ensure messages queued in-flight are not sent and hence, avoid re-ordering + // This works because there is only 1 IO thread and no IO can be done until the callback returns. + // Do not create a new producer here! It cannot be done without data loss for all concurrency modes. + fatalException.compareAndSet(null, producerException) + try { + currentProducer.close(0, TimeUnit.MILLISECONDS) + } catch { + case exception: Exception => error("Exception while closing producer.", exception) + } + } + } + + /** + * @return the current producer. Never returns null. + */ + private def getOrCreateCurrentProducer = { + var currentProducer = producerRef.get + + if (currentProducer == null) { + if (dropProducerExceptions && !stopped) { + // Note: While this lock prevents others from creating a new producer, they could still set it to null. + producerCreationLock.synchronized { + currentProducer = producerRef.get + if (currentProducer == null) { + currentProducer = getProducer() + producerRef.set(currentProducer) + } + } + // Invariant: currentProducer must not be null at this point. + } else { + throw new SystemProducerException("Kafka producer is null.") + } + } + currentProducer + } +} diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemProducerMetrics.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemProducerMetrics.scala new file mode 100644 index 0000000000..e3ff90ab09 --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemProducerMetrics.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.system.kafka_deprecated + +import org.apache.samza.metrics.{MetricsHelper, MetricsRegistry, MetricsRegistryMap} + +class KafkaSystemProducerMetrics(val systemName: String = "unknown", val registry: MetricsRegistry = new MetricsRegistryMap) extends MetricsHelper { + /* Tracks the number of calls made to send in KafkaSystemProducer */ + val sends = newCounter("producer-sends") + /* Tracks the number of calls made to flush in KafkaSystemProducer */ + val flushes = newCounter("flushes") + /* Tracks how long the flush call takes to complete */ + val flushNs = newTimer("flush-ns") + /* Tracks the number of times the system producer retries a send request (due to RetriableException) */ + @Deprecated + val retries = newCounter("producer-retries") + /* Tracks the number of times flush operation failed */ + val flushFailed = newCounter("flush-failed") + /* Tracks the number of send requests that was failed by the KafkaProducer (due to unrecoverable errors) */ + val sendFailed = newCounter("producer-send-failed") + /* Tracks the number of send requests that was successfully completed by the KafkaProducer */ + val sendSuccess = newCounter("producer-send-success") + + override def getPrefix = systemName + "-" +} diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/MessageSink.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/MessageSink.scala new file mode 100644 index 0000000000..d63f805450 --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/MessageSink.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.system.kafka_deprecated + +import kafka.common.TopicAndPartition +import kafka.message.MessageAndOffset + +private[kafka_deprecated] trait MessageSink { + def setIsAtHighWatermark(tp: TopicAndPartition, isAtHighWatermark: Boolean): Unit + + def addMessage(tp: TopicAndPartition, msg: MessageAndOffset, highWatermark: Long): Unit + + def abdicate(tp: TopicAndPartition, nextOffset: Long): Unit + + def refreshDropped(): Unit + + def needsMoreMessages(tp: TopicAndPartition): Boolean +} diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/TopicMetadataCache.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/TopicMetadataCache.scala new file mode 100644 index 0000000000..fbfd1c6641 --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/TopicMetadataCache.scala @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.system.kafka_deprecated + +import org.apache.samza.util.Logging +import kafka.api.TopicMetadata +import org.apache.samza.util.KafkaUtil + +/** + * TopicMetadataCache is used to cache all the topic metadata for Kafka per + * (system, topic) partition. The cache access is thread safe. Each entry in + * the cache is refreshed after a specified interval. The cache uses the passed + * in getTopicInfoFromStore that retrieves the topic metadata from the store (usually zookeeper). + */ +object TopicMetadataCache extends Logging { + private case class MetadataInfo(var streamMetadata: TopicMetadata, var lastRefreshMs: Long) + private val topicMetadataMap: scala.collection.mutable.Map[(String, String), MetadataInfo] = new scala.collection.mutable.HashMap[(String, String), MetadataInfo] + private val lock = new Object + + // used to fetch the topic metadata from the store. Accepts a topic and system + type FetchTopicMetadataType = (Set[String]) => Map[String, TopicMetadata] + + def getTopicMetadata(topics: Set[String], systemName: String, getTopicInfoFromStore: FetchTopicMetadataType, cacheTimeout: Long = 5000L, getTime: () => Long = { System.currentTimeMillis }): Map[String, TopicMetadata] = { + lock synchronized { + val time = getTime() + val missingTopics = topics.filter(topic => !topicMetadataMap.contains(systemName, topic)) + val topicsWithBadOrExpiredMetadata = (topics -- missingTopics).filter(topic => { + val metadata = topicMetadataMap(systemName, topic) + hasBadErrorCode(metadata.streamMetadata) || ((time - metadata.lastRefreshMs) > cacheTimeout) + }) + val topicsToRefresh = missingTopics ++ topicsWithBadOrExpiredMetadata + + if (topicsToRefresh.size > 0) { + // Refresh topic information for any missing, expired, or bad topic metadata. + topicMetadataMap ++= getTopicInfoFromStore(missingTopics ++ topicsWithBadOrExpiredMetadata) + .map { case (topic, metadata) => ((systemName, topic), MetadataInfo(metadata, getTime())) } + .toMap + } + + // Use our new updated cache to return a map of topic -> metadata + topicMetadataMap + .filterKeys(topics.map(topic => (systemName, topic))) + .map { + case ((systemName, topic), metadata) => + (topic, metadata.streamMetadata) + }.toMap + } + } + + def clear { + topicMetadataMap.clear + } + + /** + * Helper method to check if a topic's metadata has a bad errorCode, or if a + * partition's metadata has a bad errorCode. + */ + def hasBadErrorCode(streamMetadata: TopicMetadata) = { + KafkaUtil.isBadErrorCode(streamMetadata.error.code()) || streamMetadata.partitionsMetadata.exists(partitionMetadata => KafkaUtil.isBadErrorCode(partitionMetadata.error.code())) + } +} diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/Toss.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/Toss.scala new file mode 100644 index 0000000000..2c3ec0fab4 --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/Toss.scala @@ -0,0 +1,28 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.samza.system.kafka_deprecated + +import org.apache.samza.SamzaException + +private[kafka_deprecated] trait Toss { + def toss(s:String) = throw new SamzaException(s) +} From f1f4ebadb6480955afb60ccf22ecd8b2716ad86e Mon Sep 17 00:00:00 2001 From: Boris S Date: Wed, 17 Oct 2018 12:27:18 -0700 Subject: [PATCH 03/18] resteore KafkaSystemFactory --- .../kafka_deprecated/KafkaSystemFactory.scala | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala index 933dd59105..f76e7cb885 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala @@ -19,11 +19,12 @@ package org.apache.samza.system.kafka_deprecated import java.util.Properties + import kafka.utils.ZkUtils import org.apache.samza.SamzaException import org.apache.samza.config.ApplicationConfig.ApplicationMode -import org.apache.samza.util.{Logging, KafkaUtil, ExponentialSleepStrategy, ClientUtilTopicMetadataStore} -import org.apache.samza.config.{KafkaConfig, ApplicationConfig, StreamConfig, Config} +import org.apache.samza.util.{ClientUtilTopicMetadataStore, ExponentialSleepStrategy, KafkaUtil, Logging} +import org.apache.samza.config._ import org.apache.samza.metrics.MetricsRegistry import org.apache.samza.config.KafkaConfig.Config2Kafka import org.apache.samza.config.TaskConfig.Config2Task @@ -46,7 +47,7 @@ object KafkaSystemFactory extends Logging { class KafkaSystemFactory extends SystemFactory with Logging { def getConsumer(systemName: String, config: Config, registry: MetricsRegistry): SystemConsumer = { - val clientId = KafkaUtil.getClientId("samza-consumer", config) + val clientId = getClientId("samza-consumer", config) val metrics = new KafkaSystemConsumerMetrics(systemName, registry) // Kind of goofy to need a producer config for consumers, but we need metadata. @@ -84,7 +85,7 @@ class KafkaSystemFactory extends SystemFactory with Logging { } def getProducer(systemName: String, config: Config, registry: MetricsRegistry): SystemProducer = { - val clientId = KafkaUtil.getClientId("samza-producer", config) + val clientId = getClientId("samza-producer", config) val injectedProps = KafkaSystemFactory.getInjectedProducerProperties(systemName, config) val producerConfig = config.getKafkaSystemProducerConfig(systemName, clientId, injectedProps) val getProducer = () => { new KafkaProducer[Array[Byte], Array[Byte]](producerConfig.getProducerProperties) } @@ -103,7 +104,7 @@ class KafkaSystemFactory extends SystemFactory with Logging { } def getAdmin(systemName: String, config: Config): SystemAdmin = { - val clientId = KafkaUtil.getClientId("samza-admin", config) + val clientId = getClientId("samza-admin", config) val producerConfig = config.getKafkaSystemProducerConfig(systemName, clientId) val bootstrapServers = producerConfig.bootsrapServers val consumerConfig = config.getKafkaSystemConsumerConfig(systemName, clientId) @@ -163,4 +164,15 @@ class KafkaSystemFactory extends SystemFactory with Logging { Map() } } + def getClientId(id: String, config: Config): String = getClientId( + id, + config.getName.getOrElse(throw new ConfigException("Missing job name.")), + config.getJobId.getOrElse("1")) + + def getClientId(id: String, jobName: String, jobId: String): String = + "%s-%s-%s" format + (id.replaceAll("[^A-Za-z0-9]", "_"), + jobName.replaceAll("[^A-Za-z0-9]", "_"), + jobId.replaceAll("[^A-Za-z0-9]", "_")) + } From 1e4014740a6da4d95c3d08eca6daf3a5bf132022 Mon Sep 17 00:00:00 2001 From: Boris S Date: Wed, 17 Oct 2018 12:31:17 -0700 Subject: [PATCH 04/18] import --- .../samza/system/kafka_deprecated/KafkaSystemFactory.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala index f76e7cb885..16aa5c2088 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala @@ -23,7 +23,7 @@ import java.util.Properties import kafka.utils.ZkUtils import org.apache.samza.SamzaException import org.apache.samza.config.ApplicationConfig.ApplicationMode -import org.apache.samza.util.{ClientUtilTopicMetadataStore, ExponentialSleepStrategy, KafkaUtil, Logging} +import org.apache.samza.util.{ClientUtilTopicMetadataStore, ExponentialSleepStrategy, Logging} import org.apache.samza.config._ import org.apache.samza.metrics.MetricsRegistry import org.apache.samza.config.KafkaConfig.Config2Kafka From abb215d1eedfaa23f6b809fab3c48540d60c09a6 Mon Sep 17 00:00:00 2001 From: Boris S Date: Wed, 17 Oct 2018 13:09:40 -0700 Subject: [PATCH 05/18] KafkaSystemConsumer --- .../kafka_deprecated/KafkaSystemConsumer.java | 371 ------------------ .../KafkaSystemConsumer.scala | 309 +++++++++++++++ 2 files changed, 309 insertions(+), 371 deletions(-) delete mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.java create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.scala diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.java b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.java deleted file mode 100644 index 6a7200c6cf..0000000000 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.java +++ /dev/null @@ -1,371 +0,0 @@ - -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -package org.apache.samza.system.kafka_deprecated; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import kafka.common.TopicAndPartition; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.TopicPartition; -import org.apache.samza.SamzaException; -import org.apache.samza.config.Config; -import org.apache.samza.config.KafkaConfig; -import org.apache.samza.config.KafkaConsumerConfig; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SystemConsumer; -import org.apache.samza.system.SystemStreamPartition; -import org.apache.samza.util.BlockingEnvelopeMap; -import org.apache.samza.util.Clock; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import scala.Option; - - -public class KafkaSystemConsumer extends BlockingEnvelopeMap implements SystemConsumer { - - private static final Logger LOG = LoggerFactory.getLogger(KafkaSystemConsumer.class); - - private static final long FETCH_THRESHOLD = 50000; - private static final long FETCH_THRESHOLD_BYTES = -1L; - - private final Consumer kafkaConsumer; - private final String systemName; - private final String clientId; - private final AtomicBoolean stopped = new AtomicBoolean(false); - private final AtomicBoolean started = new AtomicBoolean(false); - private final Config config; - private final boolean fetchThresholdBytesEnabled; - private final KafkaSystemConsumerMetrics metrics; - - // This sink is used to transfer the messages from the proxy/consumer to the BlockingEnvelopeMap. - final KafkaConsumerMessageSink messageSink; - - // This proxy contains a separate thread, which reads kafka messages (with consumer.poll()) and populates - // BlockingEnvelopMap's buffers. - final private KafkaConsumerProxy proxy; - - // keep registration data until the start - mapping between registered SSPs and topicPartitions, and their offsets - final Map topicPartitionsToOffset = new HashMap<>(); - final Map topicPartitionsToSSP = new HashMap<>(); - - long perPartitionFetchThreshold; - long perPartitionFetchThresholdBytes; - - /** - * Create a KafkaSystemConsumer for the provided {@code systemName} - * @param systemName system name for which we create the consumer - * @param config application config - * @param metrics metrics for this KafkaSystemConsumer - * @param clock system clock - */ - public KafkaSystemConsumer(Consumer kafkaConsumer, String systemName, Config config, String clientId, - KafkaSystemConsumerMetrics metrics, Clock clock) { - - super(metrics.registry(), clock, metrics.getClass().getName()); - - this.kafkaConsumer = kafkaConsumer; - this.clientId = clientId; - this.systemName = systemName; - this.config = config; - this.metrics = metrics; - - fetchThresholdBytesEnabled = new KafkaConfig(config).isConsumerFetchThresholdBytesEnabled(systemName); - - // create a sink for passing the messages between the proxy and the consumer - messageSink = new KafkaConsumerMessageSink(); - - // Create the proxy to do the actual message reading. - String metricName = String.format("%s", systemName); - proxy = new KafkaConsumerProxy(kafkaConsumer, systemName, clientId, messageSink, metrics, metricName); - LOG.info("{}: Created KafkaConsumerProxy {} ", this, proxy); - } - - /** - * Create internal kafka consumer object, which will be used in the Proxy. - * @param systemName system name for which we create the consumer - * @param clientId client id to use int the kafka client - * @param config config - * @return kafka consumer object - */ - public static KafkaConsumer getKafkaConsumerImpl(String systemName, String clientId, Config config) { - - // extract kafka client configs - KafkaConsumerConfig consumerConfig = KafkaConsumerConfig.getKafkaSystemConsumerConfig(config, systemName, clientId); - - LOG.info("{}: KafkaClient properties {}", systemName, consumerConfig); - - return new KafkaConsumer(consumerConfig); - } - - @Override - public void start() { - if (!started.compareAndSet(false, true)) { - LOG.warn("{}: Attempting to start the consumer for the second (or more) time.", this); - return; - } - if (stopped.get()) { - LOG.error("{}: Attempting to start a stopped consumer", this); - return; - } - // initialize the subscriptions for all the registered TopicPartitions - startSubscription(); - // needs to be called after all the registrations are completed - setFetchThresholds(); - - startConsumer(); - LOG.info("{}: Consumer started", this); - } - - private void startSubscription() { - //subscribe to all the registered TopicPartitions - LOG.info("{}: Consumer subscribes to {}", this, topicPartitionsToSSP.keySet()); - try { - synchronized (kafkaConsumer) { - // we are using assign (and not subscribe), so we need to specify both topic and partition - kafkaConsumer.assign(topicPartitionsToSSP.keySet()); - } - } catch (Exception e) { - throw new SamzaException("Consumer subscription failed for " + this, e); - } - } - - /** - * Set the offsets to start from. - * Register the TopicPartitions with the proxy. - * Start the proxy. - */ - void startConsumer() { - // set the offset for each TopicPartition - if (topicPartitionsToOffset.size() <= 0) { - LOG.error ("{}: Consumer is not subscribed to any SSPs", this); - } - - topicPartitionsToOffset.forEach((tp, startingOffsetString) -> { - long startingOffset = Long.valueOf(startingOffsetString); - - try { - synchronized (kafkaConsumer) { - kafkaConsumer.seek(tp, startingOffset); // this value should already be the 'upcoming' value - } - } catch (Exception e) { - // all recoverable execptions are handled by the client. - // if we get here there is nothing left to do but bail out. - String msg = - String.format("%s: Got Exception while seeking to %s for partition %s", this, startingOffsetString, tp); - LOG.error(msg, e); - throw new SamzaException(msg, e); - } - - LOG.info("{}: Changing consumer's starting offset for tp = %s to %s", this, tp, startingOffsetString); - - // add the partition to the proxy - proxy.addTopicPartition(topicPartitionsToSSP.get(tp), startingOffset); - }); - - // start the proxy thread - if (proxy != null && !proxy.isRunning()) { - LOG.info("{}: Starting proxy {}", this, proxy); - proxy.start(); - } - } - - private void setFetchThresholds() { - // get the thresholds, and set defaults if not defined. - KafkaConfig kafkaConfig = new KafkaConfig(config); - - Option fetchThresholdOption = kafkaConfig.getConsumerFetchThreshold(systemName); - long fetchThreshold = FETCH_THRESHOLD; - if (fetchThresholdOption.isDefined()) { - fetchThreshold = Long.valueOf(fetchThresholdOption.get()); - } - - Option fetchThresholdBytesOption = kafkaConfig.getConsumerFetchThresholdBytes(systemName); - long fetchThresholdBytes = FETCH_THRESHOLD_BYTES; - if (fetchThresholdBytesOption.isDefined()) { - fetchThresholdBytes = Long.valueOf(fetchThresholdBytesOption.get()); - } - - int numPartitions = topicPartitionsToSSP.size(); - if (numPartitions != topicPartitionsToOffset.size()) { - throw new SamzaException("topicPartitionsToSSP.size() doesn't match topicPartitionsToOffset.size()"); - } - - - if (numPartitions > 0) { - perPartitionFetchThreshold = fetchThreshold / numPartitions; - if (fetchThresholdBytesEnabled) { - // currently this feature cannot be enabled, because we do not have the size of the messages available. - // messages get double buffered, hence divide by 2 - perPartitionFetchThresholdBytes = (fetchThresholdBytes / 2) / numPartitions; - } - } - LOG.info("{}: fetchThresholdBytes = {}; fetchThreshold={}; numPartitions={}, perPartitionFetchThreshold={}, perPartitionFetchThresholdBytes(0 if disabled)={}", - this, fetchThresholdBytes, fetchThreshold, numPartitions, perPartitionFetchThreshold, perPartitionFetchThresholdBytes); - } - - @Override - public void stop() { - if (!stopped.compareAndSet(false, true)) { - LOG.warn("{}: Attempting to stop stopped consumer.", this); - return; - } - - LOG.info("{}: Stopping Samza kafkaConsumer ", this); - - // stop the proxy (with 1 minute timeout) - if (proxy != null) { - LOG.info("{}: Stopping proxy {}", this, proxy); - proxy.stop(TimeUnit.SECONDS.toMillis(60)); - } - - try { - synchronized (kafkaConsumer) { - LOG.info("{}: Closing kafkaSystemConsumer {}", this, kafkaConsumer); - kafkaConsumer.close(); - } - } catch (Exception e) { - LOG.warn("{}: Failed to stop KafkaSystemConsumer.", this, e); - } - } - - /** - * record the ssp and the offset. Do not submit it to the consumer yet. - * @param systemStreamPartition ssp to register - * @param offset offset to register with - */ - @Override - public void register(SystemStreamPartition systemStreamPartition, String offset) { - if (started.get()) { - String msg = String.format("%s: Trying to register partition after consumer has been started. ssp=%s", this, - systemStreamPartition); - throw new SamzaException(msg); - } - - if (!systemStreamPartition.getSystem().equals(systemName)) { - LOG.warn("{}: ignoring SSP {}, because this consumer's system doesn't match.", this, systemStreamPartition); - return; - } - LOG.info("{}: Registering ssp = {} with offset {}", this, systemStreamPartition, offset); - - super.register(systemStreamPartition, offset); - - TopicPartition tp = toTopicPartition(systemStreamPartition); - - topicPartitionsToSSP.put(tp, systemStreamPartition); - - String existingOffset = topicPartitionsToOffset.get(tp); - // register the older (of the two) offset in the consumer, to guarantee we do not miss any messages. - if (existingOffset == null || compareOffsets(existingOffset, offset) > 0) { - topicPartitionsToOffset.put(tp, offset); - } - - metrics.registerTopicAndPartition(toTopicAndPartition(tp)); - } - - /** - * Compare two String offsets. - * Note. There is a method in KafkaSystemAdmin that does that, but that would require instantiation of systemadmin for each consumer. - * @return see {@link Long#compareTo(Long)} - */ - private static int compareOffsets(String offset1, String offset2) { - return Long.valueOf(offset1).compareTo(Long.valueOf(offset2)); - } - - @Override - public String toString() { - return String.format("%s:%s", systemName, clientId); - } - - @Override - public Map> poll( - Set systemStreamPartitions, long timeout) throws InterruptedException { - - // check if the proxy is running - if (!proxy.isRunning()) { - stop(); - String message = String.format("%s: KafkaConsumerProxy has stopped.", this); - throw new SamzaException(message, proxy.getFailureCause()); - } - - return super.poll(systemStreamPartitions, timeout); - } - - /** - * convert from TopicPartition to TopicAndPartition - */ - public static TopicAndPartition toTopicAndPartition(TopicPartition tp) { - return new TopicAndPartition(tp.topic(), tp.partition()); - } - - /** - * convert to TopicPartition from SystemStreamPartition - */ - public static TopicPartition toTopicPartition(SystemStreamPartition ssp) { - return new TopicPartition(ssp.getStream(), ssp.getPartition().getPartitionId()); - } - - /** - * return system name for this consumer - * @return system name - */ - public String getSystemName() { - return systemName; - } - - public class KafkaConsumerMessageSink { - - public void setIsAtHighWatermark(SystemStreamPartition ssp, boolean isAtHighWatermark) { - setIsAtHead(ssp, isAtHighWatermark); - } - - boolean needsMoreMessages(SystemStreamPartition ssp) { - LOG.debug("{}: needsMoreMessages from following SSP: {}. fetchLimitByBytes enabled={}; messagesSizeInQueue={};" - + "(limit={}); messagesNumInQueue={}(limit={};", this, ssp, fetchThresholdBytesEnabled, - getMessagesSizeInQueue(ssp), perPartitionFetchThresholdBytes, getNumMessagesInQueue(ssp), - perPartitionFetchThreshold); - - if (fetchThresholdBytesEnabled) { - return getMessagesSizeInQueue(ssp) < perPartitionFetchThresholdBytes; - } else { - return getNumMessagesInQueue(ssp) < perPartitionFetchThreshold; - } - } - - void addMessage(SystemStreamPartition ssp, IncomingMessageEnvelope envelope) { - LOG.trace("{}: Incoming message ssp = {}: envelope = {}.", this, ssp, envelope); - - try { - put(ssp, envelope); - } catch (InterruptedException e) { - throw new SamzaException( - String.format("%s: Consumer was interrupted while trying to add message with offset %s for ssp %s", this, - envelope.getOffset(), ssp)); - } - } - } -} diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.scala new file mode 100644 index 0000000000..dde15ad134 --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.scala @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.samza.system.kafka_deprecated + +import kafka.common.TopicAndPartition +import org.apache.samza.util.Logging +import kafka.message.Message +import kafka.message.MessageAndOffset +import org.apache.samza.Partition +import org.apache.kafka.common.utils.Utils +import org.apache.samza.util.Clock +import kafka.serializer.DefaultDecoder +import kafka.serializer.Decoder +import org.apache.samza.util.BlockingEnvelopeMap +import org.apache.samza.system.SystemStreamPartition +import org.apache.samza.system.IncomingMessageEnvelope +import kafka.consumer.ConsumerConfig +import org.apache.samza.util.TopicMetadataStore +import kafka.api.PartitionMetadata +import kafka.api.TopicMetadata +import org.apache.samza.util.ExponentialSleepStrategy +import java.util.concurrent.ConcurrentHashMap +import scala.collection.JavaConverters._ +import org.apache.samza.system.SystemAdmin + +object KafkaSystemConsumer { + + // Approximate additional shallow heap overhead per message in addition to the raw bytes + // received from Kafka 4 + 64 + 4 + 4 + 4 = 80 bytes overhead. + // As this overhead is a moving target, and not very large + // compared to the message size its being ignore in the computation for now. + val MESSAGE_SIZE_OVERHEAD = 4 + 64 + 4 + 4 + 4; + + def toTopicAndPartition(systemStreamPartition: SystemStreamPartition) = { + val topic = systemStreamPartition.getStream + val partitionId = systemStreamPartition.getPartition.getPartitionId + TopicAndPartition(topic, partitionId) + } +} + +/** + * Maintain a cache of BrokerProxies, returning the appropriate one for the + * requested topic and partition. + */ +private[kafka] class KafkaSystemConsumer( + systemName: String, + systemAdmin: SystemAdmin, + metrics: KafkaSystemConsumerMetrics, + metadataStore: TopicMetadataStore, + clientId: String, + timeout: Int = ConsumerConfig.ConsumerTimeoutMs, + bufferSize: Int = ConsumerConfig.SocketBufferSize, + fetchSize: StreamFetchSizes = new StreamFetchSizes, + consumerMinSize: Int = ConsumerConfig.MinFetchBytes, + consumerMaxWait: Int = ConsumerConfig.MaxFetchWaitMs, + + /** + * Defines a low water mark for how many messages we buffer before we start + * executing fetch requests against brokers to get more messages. This value + * is divided equally among all registered SystemStreamPartitions. For + * example, if fetchThreshold is set to 50000, and there are 50 + * SystemStreamPartitions registered, then the per-partition threshold is + * 1000. As soon as a SystemStreamPartition's buffered message count drops + * below 1000, a fetch request will be executed to get more data for it. + * + * Increasing this parameter will decrease the latency between when a queue + * is drained of messages and when new messages are enqueued, but also leads + * to an increase in memory usage since more messages will be held in memory. + */ + fetchThreshold: Int = 50000, + /** + * Defines a low water mark for how many bytes we buffer before we start + * executing fetch requests against brokers to get more messages. This + * value is divided by 2 because the messages are buffered twice, once in + * KafkaConsumer and then in SystemConsumers. This value + * is divided equally among all registered SystemStreamPartitions. + * However this is a soft limit per partition, as the + * bytes are cached at the message boundaries, and the actual usage can be + * 1000 bytes + size of max message in the partition for a given stream. + * The bytes if the size of the bytebuffer in Message. Hence, the + * Object overhead is not taken into consideration. In this codebase + * it seems to be quite small. Hence, even for 500000 messages this is around 4MB x 2 = 8MB, + * which is not considerable. + * + * For example, + * if fetchThresholdBytes is set to 100000 bytes, and there are 50 + * SystemStreamPartitions registered, then the per-partition threshold is + * (100000 / 2) / 50 = 1000 bytes. + * As this is a soft limit, the actual usage can be 1000 bytes + size of max message. + * As soon as a SystemStreamPartition's buffered messages bytes drops + * below 1000, a fetch request will be executed to get more data for it. + * + * Increasing this parameter will decrease the latency between when a queue + * is drained of messages and when new messages are enqueued, but also leads + * to an increase in memory usage since more messages will be held in memory. + * + * The default value is -1, which means this is not used. When the value + * is > 0, then the fetchThreshold which is count based is ignored. + */ + fetchThresholdBytes: Long = -1, + /** + * if(fetchThresholdBytes > 0) true else false + */ + fetchLimitByBytesEnabled: Boolean = false, + offsetGetter: GetOffset = new GetOffset("fail"), + deserializer: Decoder[Object] = new DefaultDecoder().asInstanceOf[Decoder[Object]], + keyDeserializer: Decoder[Object] = new DefaultDecoder().asInstanceOf[Decoder[Object]], + retryBackoff: ExponentialSleepStrategy = new ExponentialSleepStrategy, + clock: () => Long = { System.currentTimeMillis }) extends BlockingEnvelopeMap( + metrics.registry, + new Clock { + def currentTimeMillis = clock() + }, + classOf[KafkaSystemConsumerMetrics].getName) with Toss with Logging { + + type HostPort = (String, Int) + val brokerProxies = scala.collection.mutable.Map[HostPort, BrokerProxy]() + val topicPartitionsAndOffsets: scala.collection.concurrent.Map[TopicAndPartition, String] = new ConcurrentHashMap[TopicAndPartition, String]().asScala + var perPartitionFetchThreshold = fetchThreshold + var perPartitionFetchThresholdBytes = 0L + + def start() { + if (topicPartitionsAndOffsets.size > 0) { + perPartitionFetchThreshold = fetchThreshold / topicPartitionsAndOffsets.size + // messages get double buffered, hence divide by 2 + if(fetchLimitByBytesEnabled) { + perPartitionFetchThresholdBytes = (fetchThresholdBytes / 2) / topicPartitionsAndOffsets.size + } + } + + systemAdmin.start() + refreshBrokers + } + + override def register(systemStreamPartition: SystemStreamPartition, offset: String) { + super.register(systemStreamPartition, offset) + + val topicAndPartition = KafkaSystemConsumer.toTopicAndPartition(systemStreamPartition) + val existingOffset = topicPartitionsAndOffsets.getOrElseUpdate(topicAndPartition, offset) + // register the older offset in the consumer + if (systemAdmin.offsetComparator(existingOffset, offset) >= 0) { + topicPartitionsAndOffsets.replace(topicAndPartition, offset) + } + + metrics.registerTopicAndPartition(KafkaSystemConsumer.toTopicAndPartition(systemStreamPartition)) + } + + def stop() { + systemAdmin.stop() + brokerProxies.values.foreach(_.stop) + } + + protected def createBrokerProxy(host: String, port: Int): BrokerProxy = { + info("Creating new broker proxy for host: %s and port: %s" format(host, port)) + new BrokerProxy(host, port, systemName, clientId, metrics, sink, timeout, bufferSize, fetchSize, consumerMinSize, consumerMaxWait, offsetGetter) + } + + protected def getPartitionMetadata(topicMetadata: TopicMetadata, partition: Int): Option[PartitionMetadata] = { + topicMetadata.partitionsMetadata.find(_.partitionId == partition) + } + + protected def getLeaderHostPort(partitionMetadata: Option[PartitionMetadata]): Option[(String, Int)] = { + // Whatever we do, we can't say Broker, even though we're + // manipulating it here. Broker is a private type and Scala doesn't seem + // to care about that as long as you don't explicitly declare its type. + val brokerOption = partitionMetadata.flatMap(_.leader) + + brokerOption match { + case Some(broker) => Some(broker.host, broker.port) + case _ => None + } + } + + def refreshBrokers { + var tpToRefresh = topicPartitionsAndOffsets.keySet.toList + info("Refreshing brokers for: %s" format topicPartitionsAndOffsets) + retryBackoff.run( + loop => { + val topics = tpToRefresh.map(_.topic).toSet + val topicMetadata = TopicMetadataCache.getTopicMetadata(topics, systemName, (topics: Set[String]) => metadataStore.getTopicInfo(topics)) + + // addTopicPartition one at a time, leaving the to-be-done list intact in case of exceptions. + // This avoids trying to re-add the same topic partition repeatedly + def refresh() = { + val head = tpToRefresh.head + // refreshBrokers can be called from abdicate and refreshDropped, + // both of which are triggered from BrokerProxy threads. To prevent + // accidentally creating multiple objects for the same broker, or + // accidentally not updating the topicPartitionsAndOffsets variable, + // we need to lock. + this.synchronized { + // Check if we still need this TopicAndPartition inside the + // critical section. If we don't, then notAValidEvent it. + topicPartitionsAndOffsets.get(head) match { + case Some(nextOffset) => + val partitionMetadata = getPartitionMetadata(topicMetadata(head.topic), head.partition) + getLeaderHostPort(partitionMetadata) match { + case Some((host, port)) => + debug("Got partition metadata for %s: %s" format(head, partitionMetadata.get)) + val brokerProxy = brokerProxies.getOrElseUpdate((host, port), createBrokerProxy(host, port)) + brokerProxy.addTopicPartition(head, Option(nextOffset)) + brokerProxy.start + debug("Claimed topic-partition (%s) for (%s)".format(head, brokerProxy)) + topicPartitionsAndOffsets -= head + case None => info("No metadata available for: %s. Will try to refresh and add to a consumer thread later." format head) + } + case _ => debug("Ignoring refresh for %s because we already added it from another thread." format head) + } + } + tpToRefresh.tail + } + + while (!tpToRefresh.isEmpty) { + tpToRefresh = refresh() + } + + loop.done + }, + + (exception, loop) => { + warn("While refreshing brokers for %s: %s. Retrying." format (tpToRefresh.head, exception)) + debug("Exception detail:", exception) + }) + } + + val sink = new MessageSink { + var lastDroppedRefresh = clock() + + def refreshDropped() { + if (topicPartitionsAndOffsets.size > 0 && clock() - lastDroppedRefresh > 10000) { + refreshBrokers + lastDroppedRefresh = clock() + } + } + + def setIsAtHighWatermark(tp: TopicAndPartition, isAtHighWatermark: Boolean) { + setIsAtHead(toSystemStreamPartition(tp), isAtHighWatermark) + } + + def needsMoreMessages(tp: TopicAndPartition) = { + if(fetchLimitByBytesEnabled) { + getMessagesSizeInQueue(toSystemStreamPartition(tp)) < perPartitionFetchThresholdBytes + } else { + getNumMessagesInQueue(toSystemStreamPartition(tp)) < perPartitionFetchThreshold + } + } + + def getMessageSize(message: Message): Integer = { + message.size + KafkaSystemConsumer.MESSAGE_SIZE_OVERHEAD + } + + def addMessage(tp: TopicAndPartition, msg: MessageAndOffset, highWatermark: Long) = { + trace("Incoming message %s: %s." format (tp, msg)) + + val systemStreamPartition = toSystemStreamPartition(tp) + val isAtHead = highWatermark == msg.offset + val offset = msg.offset.toString + val key = if (msg.message.key != null) { + keyDeserializer.fromBytes(Utils.readBytes(msg.message.key)) + } else { + null + } + val message = if (!msg.message.isNull) { + deserializer.fromBytes(Utils.readBytes(msg.message.payload)) + } else { + null + } + + if(fetchLimitByBytesEnabled ) { + val ime = new IncomingMessageEnvelope(systemStreamPartition, offset, key, message, getMessageSize(msg.message)) + ime.setTimestamp(if (!msg.message.isNull) msg.message.timestamp else 0L) + put(systemStreamPartition, ime) + } else { + val ime = new IncomingMessageEnvelope(systemStreamPartition, offset, key, message) + ime.setTimestamp(if (!msg.message.isNull) msg.message.timestamp else 0L) + put(systemStreamPartition, ime) + } + + setIsAtHead(systemStreamPartition, isAtHead) + } + + def abdicate(tp: TopicAndPartition, nextOffset: Long) { + info("Abdicating for %s" format (tp)) + topicPartitionsAndOffsets += tp -> nextOffset.toString + refreshBrokers + } + + private def toSystemStreamPartition(tp: TopicAndPartition) = { + new SystemStreamPartition(systemName, tp.topic, new Partition(tp.partition)) + } + } +} From de5385e96987cc3136c0af07419152d0dacac684 Mon Sep 17 00:00:00 2001 From: Boris S Date: Wed, 17 Oct 2018 17:15:42 -0700 Subject: [PATCH 06/18] BrokerProxy --- .../samza/system/kafka_deprecated/KafkaSystemConsumer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.scala index dde15ad134..b7c436839a 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumer.scala @@ -59,7 +59,7 @@ object KafkaSystemConsumer { * Maintain a cache of BrokerProxies, returning the appropriate one for the * requested topic and partition. */ -private[kafka] class KafkaSystemConsumer( +private[kafka_deprecated] class KafkaSystemConsumer( systemName: String, systemAdmin: SystemAdmin, metrics: KafkaSystemConsumerMetrics, From b3b4183f169b7dad816762065131e6fbcb724a3b Mon Sep 17 00:00:00 2001 From: Boris S Date: Wed, 17 Oct 2018 17:29:02 -0700 Subject: [PATCH 07/18] KafkaSystemConsumerMetrics --- .../system/kafka_deprecated/BrokerProxy.scala | 332 ++++++++++++++++++ .../KafkaSystemConsumerMetrics.scala | 68 ++-- .../kafka_deprecated/KafkaSystemFactory.scala | 6 +- 3 files changed, 371 insertions(+), 35 deletions(-) create mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/BrokerProxy.scala diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/BrokerProxy.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/BrokerProxy.scala new file mode 100644 index 0000000000..a39752b74d --- /dev/null +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/BrokerProxy.scala @@ -0,0 +1,332 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.samza.system.kafka_deprecated + +import java.lang.Thread.UncaughtExceptionHandler +import java.nio.channels.ClosedByInterruptException +import java.util.concurrent.{ConcurrentHashMap, CountDownLatch} + +import kafka.api._ +import kafka.common.{ErrorMapping, NotLeaderForPartitionException, TopicAndPartition, UnknownTopicOrPartitionException} +import kafka.consumer.ConsumerConfig +import kafka.message.MessageSet +import org.apache.samza.SamzaException +import org.apache.samza.util.ExponentialSleepStrategy +import org.apache.samza.util.KafkaUtil +import org.apache.samza.util.Logging + +import scala.collection.JavaConverters._ +import scala.collection.concurrent + +/** + * A BrokerProxy consolidates Kafka fetches meant for a particular broker and retrieves them all at once, providing + * a way for consumers to retrieve those messages by topic and partition. + */ +class BrokerProxy( + val host: String, + val port: Int, + val system: String, + val clientID: String, + val metrics: KafkaSystemConsumerMetrics, + val messageSink: MessageSink, + val timeout: Int = ConsumerConfig.SocketTimeout, + val bufferSize: Int = ConsumerConfig.SocketBufferSize, + val fetchSize: StreamFetchSizes = new StreamFetchSizes, + val consumerMinSize:Int = ConsumerConfig.MinFetchBytes, + val consumerMaxWait:Int = ConsumerConfig.MaxFetchWaitMs, + offsetGetter: GetOffset = new GetOffset("fail")) extends Toss with Logging { + + /** + * How long should the fetcher thread sleep before checking if any TopicPartitions has been added to its purview + */ + val sleepMSWhileNoTopicPartitions = 100 + + /** What's the next offset for a particular partition? **/ + val nextOffsets:concurrent.Map[TopicAndPartition, Long] = new ConcurrentHashMap[TopicAndPartition, Long]().asScala + + /** Block on the first call to get message if the fetcher has not yet returned its initial results **/ + // TODO: It should be sufficient to just use the count down latch and await on it for each of the calls, but + // VisualVM was showing the consumer thread spending all its time in the await method rather than returning + // immediately, even though the process was proceeding normally. Hence the extra boolean. Should be investigated. + val firstCallBarrier = new CountDownLatch(1) + var firstCall = true + + var simpleConsumer = createSimpleConsumer() + + metrics.registerBrokerProxy(host, port) + + def createSimpleConsumer() = { + val hostString = "%s:%d" format (host, port) + info("Creating new SimpleConsumer for host %s for system %s" format (hostString, system)) + + val sc = new DefaultFetchSimpleConsumer(host, port, timeout, bufferSize, clientID, fetchSize, consumerMinSize, consumerMaxWait) + sc + } + + def addTopicPartition(tp: TopicAndPartition, nextOffset: Option[String]) = { + debug("Adding new topic and partition %s to queue for %s" format (tp, host)) + + if (nextOffsets.asJava.containsKey(tp)) { + toss("Already consuming TopicPartition %s" format tp) + } + + val offset = if (nextOffset.isDefined && offsetGetter.isValidOffset(simpleConsumer, tp, nextOffset.get)) { + nextOffset + .get + .toLong + } else { + warn("It appears that we received an invalid or empty offset %s for %s. Attempting to use Kafka's auto.offset.reset setting. This can result in data loss if processing continues." format (nextOffset, tp)) + + offsetGetter.getResetOffset(simpleConsumer, tp) + } + + debug("Got offset %s for new topic and partition %s." format (offset, tp)) + + nextOffsets += tp -> offset + + metrics.topicPartitions.get((host, port)).set(nextOffsets.size) + } + + def removeTopicPartition(tp: TopicAndPartition) = { + if (nextOffsets.asJava.containsKey(tp)) { + val offset = nextOffsets.remove(tp) + metrics.topicPartitions.get((host, port)).set(nextOffsets.size) + debug("Removed %s" format tp) + offset + } else { + warn("Asked to remove topic and partition %s, but not in map (keys = %s)" format (tp, nextOffsets.keys.mkString(","))) + None + } + } + + val thread = new Thread(new Runnable { + def run { + var reconnect = false + + try { + (new ExponentialSleepStrategy).run( + loop => { + if (reconnect) { + metrics.reconnects.get((host, port)).inc + simpleConsumer.close() + simpleConsumer = createSimpleConsumer() + } + + while (!Thread.currentThread.isInterrupted) { + messageSink.refreshDropped + if (nextOffsets.size == 0) { + debug("No TopicPartitions to fetch. Sleeping.") + Thread.sleep(sleepMSWhileNoTopicPartitions) + } else { + fetchMessages + + // If we got here, fetchMessages didn't throw an exception, i.e. it was successful. + // In that case, reset the loop delay, so that the next time an error occurs, + // we start with a short retry delay. + loop.reset + } + } + }, + + (exception, loop) => { + warn("Restarting consumer due to %s. Releasing ownership of all partitions, and restarting consumer. Turn on debugging to get a full stack trace." format exception) + debug("Exception detail:", exception) + abdicateAll + reconnect = true + }) + } catch { + case e: InterruptedException => info("Got interrupt exception in broker proxy thread.") + case e: ClosedByInterruptException => info("Got closed by interrupt exception in broker proxy thread.") + case e: OutOfMemoryError => throw new SamzaException("Got out of memory error in broker proxy thread.") + case e: StackOverflowError => throw new SamzaException("Got stack overflow error in broker proxy thread.") + } + + if (Thread.currentThread.isInterrupted) info("Shutting down due to interrupt.") + } + }, "BrokerProxy thread pointed at %s:%d for client %s" format (host, port, clientID)) + + private def fetchMessages(): Unit = { + val topicAndPartitionsToFetch = nextOffsets.filterKeys(messageSink.needsMoreMessages(_)).toList + + if (topicAndPartitionsToFetch.size > 0) { + metrics.brokerReads.get((host, port)).inc + val response: FetchResponse = simpleConsumer.defaultFetch(topicAndPartitionsToFetch: _*) + firstCall = false + firstCallBarrier.countDown() + + // Split response into errors and non errors, processing the errors first + val (nonErrorResponses, errorResponses) = response.data.toSet.partition(_._2.error.code() == ErrorMapping.NoError) + + handleErrors(errorResponses, response) + + nonErrorResponses.foreach { case (tp, data) => moveMessagesToTheirQueue(tp, data) } + } else { + refreshLatencyMetrics + + debug("No topic/partitions need to be fetched for %s:%s right now. Sleeping %sms." format (host, port, sleepMSWhileNoTopicPartitions)) + + metrics.brokerSkippedFetchRequests.get((host, port)).inc + + Thread.sleep(sleepMSWhileNoTopicPartitions) + } + } + + /** + * Releases ownership for a single TopicAndPartition. The + * KafkaSystemConsumer will try and find a new broker for the + * TopicAndPartition. + */ + def abdicate(tp: TopicAndPartition) = removeTopicPartition(tp) match { + // Need to be mindful of a tp that was removed by another thread + case Some(offset) => messageSink.abdicate(tp, offset) + case None => warn("Tried to abdicate for topic partition not in map. Removed in interim?") + } + + /** + * Releases all TopicAndPartition ownership for this BrokerProxy thread. The + * KafkaSystemConsumer will try and find a new broker for the + * TopicAndPartition. + */ + def abdicateAll { + info("Abdicating all topic partitions.") + val immutableNextOffsetsCopy = nextOffsets.toMap + immutableNextOffsetsCopy.keySet.foreach(abdicate(_)) + } + + def handleErrors(errorResponses: Set[(TopicAndPartition, FetchResponsePartitionData)], response: FetchResponse) = { + // FetchResponse should really return Option and a list of the errors so we don't have to find them ourselves + case class Error(tp: TopicAndPartition, code: Short, exception: Exception) + + // Now subdivide the errors into three types: non-recoverable, not leader (== abdicate) and offset out of range (== get new offset) + + // Convert FetchResponse into easier-to-work-with Errors + val errors = for ( + (topicAndPartition, responseData) <- errorResponses; + error <- Option(response.error(topicAndPartition.topic, topicAndPartition.partition)) // Scala's being cranky about referring to error.getKey values... + ) yield new Error(topicAndPartition, error.code(), error.exception()) + + val (notLeaderOrUnknownTopic, otherErrors) = errors.partition { case (e) => e.code == ErrorMapping.NotLeaderForPartitionCode || e.code == ErrorMapping.UnknownTopicOrPartitionCode } + val (offsetOutOfRangeErrors, remainingErrors) = otherErrors.partition(_.code == ErrorMapping.OffsetOutOfRangeCode) + + // Can recover from two types of errors: not leader (go find the new leader) and offset out of range (go get the new offset) + // However, we want to bail as quickly as possible if there are non recoverable errors so that the state of the other + // topic-partitions remains the same. That way, when we've rebuilt the simple consumer, we can come around and + // handle the recoverable errors. + remainingErrors.foreach(e => { + warn("Got non-recoverable error codes during multifetch. Throwing an exception to trigger reconnect. Errors: %s" format remainingErrors.mkString(",")) + KafkaUtil.maybeThrowException(e.exception) }) + + notLeaderOrUnknownTopic.foreach(e => { + warn("Received (UnknownTopicOr|NotLeaderFor)Partition exception %s for %s. Abdicating" format(e.code, e.tp)) + abdicate(e.tp) + }) + + offsetOutOfRangeErrors.foreach(e => { + warn("Received OffsetOutOfRange exception for %s. Current offset = %s" format (e.tp, nextOffsets.getOrElse(e.tp, "not found in map, likely removed in the interim"))) + + try { + val newOffset = offsetGetter.getResetOffset(simpleConsumer, e.tp) + // Put the new offset into the map (if the tp still exists). Will catch it on the next go-around + nextOffsets.replace(e.tp, newOffset) + } catch { + // UnknownTopic or NotLeader are routine events and handled via abdication. All others, bail. + case _ @ (_:UnknownTopicOrPartitionException | _: NotLeaderForPartitionException) => warn("Received (UnknownTopicOr|NotLeaderFor)Partition exception %s for %s. Abdicating" format(e.code, e.tp)) + abdicate(e.tp) + } + }) + } + + def moveMessagesToTheirQueue(tp: TopicAndPartition, data: FetchResponsePartitionData) = { + val messageSet: MessageSet = data.messages + var nextOffset = nextOffsets(tp) + + messageSink.setIsAtHighWatermark(tp, data.hw == 0 || data.hw == nextOffset) + require(messageSet != null) + for (message <- messageSet.iterator) { + messageSink.addMessage(tp, message, data.hw) // TODO: Verify this is correct + + nextOffset = message.nextOffset + + val bytesSize = message.message.payloadSize + message.message.keySize + metrics.reads.get(tp).inc + metrics.bytesRead.get(tp).inc(bytesSize) + metrics.brokerBytesRead.get((host, port)).inc(bytesSize) + metrics.offsets.get(tp).set(nextOffset) + } + + nextOffsets.replace(tp, nextOffset) // use replace rather than put in case this tp was removed while we were fetching. + + // Update high water mark + val hw = data.hw + if (hw >= 0) { + metrics.highWatermark.get(tp).set(hw) + metrics.lag.get(tp).set(hw - nextOffset) + } else { + debug("Got a high water mark less than 0 (%d) for %s, so skipping." format (hw, tp)) + } + } + override def toString() = "BrokerProxy for %s:%d" format (host, port) + + def start { + if (!thread.isAlive) { + info("Starting " + toString) + thread.setDaemon(true) + thread.setName("Samza BrokerProxy " + thread.getName) + thread.setUncaughtExceptionHandler(new UncaughtExceptionHandler { + override def uncaughtException(t: Thread, e: Throwable) = error("Uncaught exception in broker proxy:", e) + }) + thread.start + } else { + debug("Tried to start an already started broker proxy (%s). Ignoring." format toString) + } + } + + def stop { + info("Shutting down " + toString) + + if (simpleConsumer != null) { + info("closing simple consumer...") + simpleConsumer.close + } + + thread.interrupt + thread.join + } + + private def refreshLatencyMetrics { + nextOffsets.foreach{ + case (topicAndPartition, offset) => { + val latestOffset = simpleConsumer.earliestOrLatestOffset(topicAndPartition, -1, Request.OrdinaryConsumerId) + trace("latest offset of %s is %s" format (topicAndPartition, latestOffset)) + if (latestOffset >= 0) { + // only update the registered topicAndpartitions + if(metrics.highWatermark.containsKey(topicAndPartition)) { + metrics.highWatermark.get(topicAndPartition).set(latestOffset) + } + if(metrics.lag.containsKey(topicAndPartition)) { + metrics.lag.get(topicAndPartition).set(latestOffset - offset) + } + } + } + } + } +} diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumerMetrics.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumerMetrics.scala index b01fecc1b9..08b1a0c4dc 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumerMetrics.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemConsumerMetrics.scala @@ -19,10 +19,13 @@ package org.apache.samza.system.kafka_deprecated +import org.apache.samza.metrics.MetricsHelper +import org.apache.samza.metrics.MetricsRegistryMap +import org.apache.samza.metrics.MetricsRegistry import java.util.concurrent.ConcurrentHashMap - import kafka.common.TopicAndPartition -import org.apache.samza.metrics._ +import org.apache.samza.metrics.Counter +import org.apache.samza.metrics.Gauge class KafkaSystemConsumerMetrics(val systemName: String = "unknown", val registry: MetricsRegistry = new MetricsRegistryMap) extends MetricsHelper { val offsets = new ConcurrentHashMap[TopicAndPartition, Counter] @@ -31,66 +34,67 @@ class KafkaSystemConsumerMetrics(val systemName: String = "unknown", val registr val lag = new ConcurrentHashMap[TopicAndPartition, Gauge[Long]] val highWatermark = new ConcurrentHashMap[TopicAndPartition, Gauge[Long]] - val clientBytesRead = new ConcurrentHashMap[String, Counter] - val clientReads = new ConcurrentHashMap[String, Counter] - val clientSkippedFetchRequests = new ConcurrentHashMap[String, Counter] - val topicPartitions = new ConcurrentHashMap[String, Gauge[Int]] + /* + * (String, Int) = (host, port) of BrokerProxy. + */ + + val reconnects = new ConcurrentHashMap[(String, Int), Counter] + val brokerBytesRead = new ConcurrentHashMap[(String, Int), Counter] + val brokerReads = new ConcurrentHashMap[(String, Int), Counter] + val brokerSkippedFetchRequests = new ConcurrentHashMap[(String, Int), Counter] + val topicPartitions = new ConcurrentHashMap[(String, Int), Gauge[Int]] def registerTopicAndPartition(tp: TopicAndPartition) = { if (!offsets.contains(tp)) { - offsets.put(tp, newCounter("%s-%s-offset-change" format(tp.topic, tp.partition))) - bytesRead.put(tp, newCounter("%s-%s-bytes-read" format(tp.topic, tp.partition))) - reads.put(tp, newCounter("%s-%s-messages-read" format(tp.topic, tp.partition))) - highWatermark.put(tp, newGauge("%s-%s-high-watermark" format(tp.topic, tp.partition), -1L)) - lag.put(tp, newGauge("%s-%s-messages-behind-high-watermark" format(tp.topic, tp.partition), 0L)) + offsets.put(tp, newCounter("%s-%s-offset-change" format (tp.topic, tp.partition))) + bytesRead.put(tp, newCounter("%s-%s-bytes-read" format (tp.topic, tp.partition))) + reads.put(tp, newCounter("%s-%s-messages-read" format (tp.topic, tp.partition))) + highWatermark.put(tp, newGauge("%s-%s-high-watermark" format (tp.topic, tp.partition), -1L)) + lag.put(tp, newGauge("%s-%s-messages-behind-high-watermark" format (tp.topic, tp.partition), 0L)) } } - def registerClientProxy(clientName: String) { - clientBytesRead.put(clientName, newCounter("%s-bytes-read" format clientName)) - clientReads.put((clientName), newCounter("%s-messages-read" format clientName)) - clientSkippedFetchRequests.put((clientName), newCounter("%s-skipped-fetch-requests" format clientName)) - topicPartitions.put(clientName, newGauge("%s-registered-topic-partitions" format clientName, 0)) + def registerBrokerProxy(host: String, port: Int) { + reconnects.put((host, port), newCounter("%s-%s-reconnects" format (host, port))) + brokerBytesRead.put((host, port), newCounter("%s-%s-bytes-read" format (host, port))) + brokerReads.put((host, port), newCounter("%s-%s-messages-read" format (host, port))) + brokerSkippedFetchRequests.put((host, port), newCounter("%s-%s-skipped-fetch-requests" format (host, port))) + topicPartitions.put((host, port), newGauge("%s-%s-topic-partitions" format (host, port), 0)) } // java friendlier interfaces // Gauges - def setNumTopicPartitions(clientName: String, value: Int) { - topicPartitions.get(clientName).set(value) + def setTopicPartitionValue(host: String, port: Int, value: Int) { + topicPartitions.get((host,port)).set(value) } - def setLagValue(topicAndPartition: TopicAndPartition, value: Long) { lag.get((topicAndPartition)).set(value); } - def setHighWatermarkValue(topicAndPartition: TopicAndPartition, value: Long) { highWatermark.get((topicAndPartition)).set(value); } // Counters - def incClientReads(clientName: String) { - clientReads.get(clientName).inc + def incBrokerReads(host: String, port: Int) { + brokerReads.get((host,port)).inc } - def incReads(topicAndPartition: TopicAndPartition) { reads.get(topicAndPartition).inc; } - def incBytesReads(topicAndPartition: TopicAndPartition, inc: Long) { bytesRead.get(topicAndPartition).inc(inc); } - - def incClientBytesReads(clientName: String, incBytes: Long) { - clientBytesRead.get(clientName).inc(incBytes) + def incBrokerBytesReads(host: String, port: Int, incBytes: Long) { + brokerBytesRead.get((host,port)).inc(incBytes) } - - def incClientSkippedFetchRequests(clientName: String) { - clientSkippedFetchRequests.get(clientName).inc() + def incBrokerSkippedFetchRequests(host: String, port: Int) { + brokerSkippedFetchRequests.get((host,port)).inc() } - def setOffsets(topicAndPartition: TopicAndPartition, offset: Long) { offsets.get(topicAndPartition).set(offset) } - + def incReconnects(host: String, port: Int) { + reconnects.get((host,port)).inc() + } override def getPrefix = systemName + "-" } diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala index 16aa5c2088..eecdbe4c5e 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaSystemFactory.scala @@ -127,7 +127,7 @@ class KafkaSystemFactory extends SystemFactory with Logging { (topicName, changelogInfo) }} - val deleteCommittedMessages = config.deleteCommittedMessages(systemName).exists(isEnabled => isEnabled.toBoolean) + val deleteCommittedMessages = config.deleteCommittedMessages(systemName) val intermediateStreamProperties: Map[String, Properties] = getIntermediateStreamProperties(config) new KafkaSystemAdmin( systemName, @@ -166,8 +166,8 @@ class KafkaSystemFactory extends SystemFactory with Logging { } def getClientId(id: String, config: Config): String = getClientId( id, - config.getName.getOrElse(throw new ConfigException("Missing job name.")), - config.getJobId.getOrElse("1")) + new JobConfig(config).getName.getOrElse(throw new ConfigException("Missing job name.")), + new JobConfig(config)getJobId) def getClientId(id: String, jobName: String, jobId: String): String = "%s-%s-%s" format From 353f903cd6f1c608367e73441df57a86c0f2976f Mon Sep 17 00:00:00 2001 From: Boris S Date: Wed, 17 Oct 2018 21:22:58 -0700 Subject: [PATCH 08/18] delete java file --- .../kafka_deprecated/KafkaConsumerProxy.java | 423 ------------------ 1 file changed, 423 deletions(-) delete mode 100644 samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaConsumerProxy.java diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaConsumerProxy.java b/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaConsumerProxy.java deleted file mode 100644 index 5ace182f77..0000000000 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka_deprecated/KafkaConsumerProxy.java +++ /dev/null @@ -1,423 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -package org.apache.samza.system.kafka_deprecated; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import kafka.common.TopicAndPartition; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.common.Metric; -import org.apache.kafka.common.MetricName; -import org.apache.kafka.common.TopicPartition; -import org.apache.samza.Partition; -import org.apache.samza.SamzaException; -import org.apache.samza.system.IncomingMessageEnvelope; -import org.apache.samza.system.SystemStreamPartition; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class contains a separate thread that reads messages from kafka and puts them into the BlockingEnvelopeMap - * through KafkaSystemConsumer.KafkaConsumerMessageSink object. - * This class is not thread safe. There will be only one instance of this class per KafkaSystemConsumer object. - * We still need some synchronization around kafkaConsumer. See pollConsumer() method for details. - */ -class KafkaConsumerProxy { - private static final Logger LOG = LoggerFactory.getLogger(KafkaConsumerProxy.class); - - private static final int SLEEP_MS_WHILE_NO_TOPIC_PARTITION = 100; - - final Thread consumerPollThread; - private final Consumer kafkaConsumer; - private final KafkaSystemConsumer.KafkaConsumerMessageSink sink; - private final KafkaSystemConsumerMetrics kafkaConsumerMetrics; - private final String metricName; - private final String systemName; - private final String clientId; - private final Map topicPartitionToSSP = new HashMap<>(); - private final Map perPartitionMetrics = new HashMap<>(); - // list of all the SSPs we poll from, with their next(most recently read + 1) offsets correspondingly. - private final Map nextOffsets = new ConcurrentHashMap<>(); - // lags behind the high water mark, as reported by the Kafka consumer. - private final Map latestLags = new HashMap<>(); - - private volatile boolean isRunning = false; - private volatile Throwable failureCause = null; - private final CountDownLatch consumerPollThreadStartLatch = new CountDownLatch(1); - - KafkaConsumerProxy(Consumer kafkaConsumer, String systemName, String clientId, - KafkaSystemConsumer.KafkaConsumerMessageSink messageSink, KafkaSystemConsumerMetrics samzaConsumerMetrics, - String metricName) { - - this.kafkaConsumer = kafkaConsumer; - this.systemName = systemName; - this.sink = messageSink; - this.kafkaConsumerMetrics = samzaConsumerMetrics; - this.metricName = metricName; - this.clientId = clientId; - - this.kafkaConsumerMetrics.registerClientProxy(metricName); - - consumerPollThread = new Thread(createProxyThreadRunnable()); - consumerPollThread.setDaemon(true); - consumerPollThread.setName( - "Samza KafkaConsumerProxy Poll " + consumerPollThread.getName() + " - " + systemName); - } - - /** - * Add new partition to the list of polled partitions. - * Bust only be called before {@link KafkaConsumerProxy#start} is called.. - */ - public void addTopicPartition(SystemStreamPartition ssp, long nextOffset) { - LOG.info(String.format("Adding new topicPartition %s with offset %s to queue for consumer %s", ssp, nextOffset, - this)); - topicPartitionToSSP.put(KafkaSystemConsumer.toTopicPartition(ssp), ssp); //registered SSPs - - // this is already vetted offset so there is no need to validate it - nextOffsets.put(ssp, nextOffset); - - kafkaConsumerMetrics.setNumTopicPartitions(metricName, nextOffsets.size()); - } - - /** - * Stop this KafkaConsumerProxy and wait for at most {@code timeoutMs}. - * @param timeoutMs maximum time to wait to stop this KafkaConsumerProxy - */ - public void stop(long timeoutMs) { - LOG.info("Shutting down KafkaConsumerProxy poll thread {} for {}", consumerPollThread.getName(), this); - - isRunning = false; - try { - consumerPollThread.join(timeoutMs/2); - // join() may timeout - // in this case we should interrupt it and wait again - if (consumerPollThread.isAlive()) { - consumerPollThread.interrupt(); - consumerPollThread.join(timeoutMs/2); - } - } catch (InterruptedException e) { - LOG.warn("Join in KafkaConsumerProxy has failed", e); - consumerPollThread.interrupt(); - } - } - - public void start() { - if (!consumerPollThread.isAlive()) { - LOG.info("Starting KafkaConsumerProxy polling thread for " + this.toString()); - - consumerPollThread.start(); - - // we need to wait until the thread starts - while (!isRunning && failureCause == null) { - try { - consumerPollThreadStartLatch.await(3000, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - LOG.info("Ignoring InterruptedException while waiting for consumer poll thread to start.", e); - } - } - } else { - LOG.warn("Tried to start an already started KafkaConsumerProxy (%s). Ignoring.", this.toString()); - } - - if (topicPartitionToSSP.size() == 0) { - String msg = String.format("Cannot start KafkaConsumerProxy without any registered TopicPartitions for %s", systemName); - LOG.error(msg); - throw new SamzaException(msg); - } - } - - boolean isRunning() { - return isRunning; - } - - Throwable getFailureCause() { - return failureCause; - } - - private void initializeLags() { - // This is expensive, so only do it once at the beginning. After the first poll, we can rely on metrics for lag. - Map endOffsets = kafkaConsumer.endOffsets(topicPartitionToSSP.keySet()); - endOffsets.forEach((tp, offset) -> { - SystemStreamPartition ssp = topicPartitionToSSP.get(tp); - long startingOffset = nextOffsets.get(ssp); - // End offsets are the offset of the newest message + 1 - // If the message we are about to consume is < end offset, we are starting with a lag. - long initialLag = endOffsets.get(tp) - startingOffset; - - LOG.info("Initial lag for SSP {} is {} (end={}, startOffset={})", ssp, initialLag, endOffsets.get(tp), startingOffset); - latestLags.put(ssp, initialLag); - sink.setIsAtHighWatermark(ssp, initialLag == 0); - }); - - // initialize lag metrics - refreshLagMetrics(); - } - - // creates a separate thread for getting the messages. - private Runnable createProxyThreadRunnable() { - Runnable runnable = () -> { - isRunning = true; - - try { - consumerPollThreadStartLatch.countDown(); - LOG.info("Starting consumer poll thread {} for system {}", consumerPollThread.getName(), systemName); - initializeLags(); - while (isRunning) { - fetchMessages(); - } - } catch (Throwable throwable) { - LOG.error(String.format("Error in KafkaConsumerProxy poll thread for system: %s.", systemName), throwable); - // KafkaSystemConsumer uses the failureCause to propagate the throwable to the container - failureCause = throwable; - isRunning = false; - } - - if (!isRunning) { - LOG.info("KafkaConsumerProxy for system {} has stopped.", systemName); - } - }; - - return runnable; - } - - private void fetchMessages() { - Set sspsToFetch = new HashSet<>(); - for (SystemStreamPartition ssp : nextOffsets.keySet()) { - if (sink.needsMoreMessages(ssp)) { - sspsToFetch.add(ssp); - } - } - LOG.debug("pollConsumer for {} SSPs: {}", sspsToFetch.size(), sspsToFetch); - if (!sspsToFetch.isEmpty()) { - kafkaConsumerMetrics.incClientReads(metricName); - - Map> response; - - response = pollConsumer(sspsToFetch, 500L); - - // move the responses into the queue - for (Map.Entry> e : response.entrySet()) { - List envelopes = e.getValue(); - if (envelopes != null) { - moveMessagesToTheirQueue(e.getKey(), envelopes); - } - } - - populateCurrentLags(sspsToFetch); // find current lags for for each SSP - } else { // nothing to read - - LOG.debug("No topic/partitions need to be fetched for system {} right now. Sleeping {}ms.", systemName, - SLEEP_MS_WHILE_NO_TOPIC_PARTITION); - - kafkaConsumerMetrics.incClientSkippedFetchRequests(metricName); - - try { - Thread.sleep(SLEEP_MS_WHILE_NO_TOPIC_PARTITION); - } catch (InterruptedException e) { - LOG.warn("Sleep in fetchMessages was interrupted"); - } - } - refreshLagMetrics(); - } - - // the actual polling of the messages from kafka - private Map> pollConsumer( - Set systemStreamPartitions, long timeoutMs) { - - // Since we need to poll only from some subset of TopicPartitions (passed as the argument), - // we need to pause the rest. - List topicPartitionsToPause = new ArrayList<>(); - List topicPartitionsToPoll = new ArrayList<>(); - - for (Map.Entry e : topicPartitionToSSP.entrySet()) { - TopicPartition tp = e.getKey(); - SystemStreamPartition ssp = e.getValue(); - if (systemStreamPartitions.contains(ssp)) { - topicPartitionsToPoll.add(tp); // consume - } else { - topicPartitionsToPause.add(tp); // ignore - } - } - - ConsumerRecords records; - try { - // Synchronize, in case the consumer is used in some other thread (metadata or something else) - synchronized (kafkaConsumer) { - // Since we are not polling from ALL the subscribed topics, so we need to "change" the subscription temporarily - kafkaConsumer.pause(topicPartitionsToPause); - kafkaConsumer.resume(topicPartitionsToPoll); - records = kafkaConsumer.poll(timeoutMs); - } - } catch (Exception e) { - // we may get InvalidOffsetException | AuthorizationException | KafkaException exceptions, - // but we still just rethrow, and log it up the stack. - LOG.error("Caught a Kafka exception in pollConsumer for system " + systemName, e); - throw e; - } - - return processResults(records); - } - - private Map> processResults(ConsumerRecords records) { - if (records == null) { - throw new SamzaException("Received null 'records' after polling consumer in KafkaConsumerProxy " + this); - } - - Map> results = new HashMap<>(records.count()); - // Parse the returned records and convert them into the IncomingMessageEnvelope. - for (ConsumerRecord record : records) { - int partition = record.partition(); - String topic = record.topic(); - TopicPartition tp = new TopicPartition(topic, partition); - - updateMetrics(record, tp); - - SystemStreamPartition ssp = topicPartitionToSSP.get(tp); - List messages = results.get(ssp); - if (messages == null) { - messages = new ArrayList<>(); - results.put(ssp, messages); - } - - K key = record.key(); - Object value = record.value(); - IncomingMessageEnvelope imEnvelope = - new IncomingMessageEnvelope(ssp, String.valueOf(record.offset()), key, value, getRecordSize(record)); - messages.add(imEnvelope); - } - if (LOG.isDebugEnabled()) { - LOG.debug("# records per SSP:"); - for (Map.Entry> e : results.entrySet()) { - List list = e.getValue(); - LOG.debug(e.getKey() + " = " + ((list == null) ? 0 : list.size())); - } - } - - return results; - } - - private int getRecordSize(ConsumerRecord r) { - int keySize = (r.key() == null) ? 0 : r.serializedKeySize(); - return keySize + r.serializedValueSize(); - } - - private void updateMetrics(ConsumerRecord r, TopicPartition tp) { - TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp); - SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition())); - - Long lag = latestLags.get(ssp); - if (lag == null) { - throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName); - } - long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark - if (currentSSPLag < 0) { - return; - } - - long recordOffset = r.offset(); - long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark - - int size = getRecordSize(r); - kafkaConsumerMetrics.incReads(tap); - kafkaConsumerMetrics.incBytesReads(tap, size); - kafkaConsumerMetrics.setOffsets(tap, recordOffset); - kafkaConsumerMetrics.incClientBytesReads(metricName, size); - kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark); - } - - private void moveMessagesToTheirQueue(SystemStreamPartition ssp, List envelopes) { - long nextOffset = nextOffsets.get(ssp); - - for (IncomingMessageEnvelope env : envelopes) { - sink.addMessage(ssp, env); // move message to the BlockingEnvelopeMap's queue - - LOG.trace("IncomingMessageEnvelope. got envelope with offset:{} for ssp={}", env.getOffset(), ssp); - nextOffset = Long.valueOf(env.getOffset()) + 1; - } - - nextOffsets.put(ssp, nextOffset); - } - - // The only way to figure out lag for the KafkaConsumer is to look at the metrics after each poll() call. - // One of the metrics (records-lag) shows how far behind the HighWatermark the consumer is. - // This method populates the lag information for each SSP into latestLags member variable. - private void populateCurrentLags(Set ssps) { - - Map consumerMetrics = kafkaConsumer.metrics(); - - // populate the MetricNames first time - if (perPartitionMetrics.isEmpty()) { - HashMap tags = new HashMap<>(); - tags.put("client-id", clientId); // this is required by the KafkaConsumer to get the metrics - - for (SystemStreamPartition ssp : ssps) { - TopicPartition tp = KafkaSystemConsumer.toTopicPartition(ssp); - perPartitionMetrics.put(ssp, new MetricName(tp + ".records-lag", "consumer-fetch-manager-metrics", "", tags)); - } - } - - for (SystemStreamPartition ssp : ssps) { - MetricName mn = perPartitionMetrics.get(ssp); - Metric currentLagMetric = consumerMetrics.get(mn); - - // High watermark is fixed to be the offset of last available message, - // so the lag is now at least 0, which is the same as Samza's definition. - // If the lag is not 0, then isAtHead is not true, and kafkaClient keeps polling. - long currentLag = (currentLagMetric != null) ? (long) currentLagMetric.value() : -1L; - latestLags.put(ssp, currentLag); - - // calls the setIsAtHead for the BlockingEnvelopeMap - sink.setIsAtHighWatermark(ssp, currentLag == 0); - } - } - - private void refreshLagMetrics() { - for (Map.Entry e : nextOffsets.entrySet()) { - SystemStreamPartition ssp = e.getKey(); - Long offset = e.getValue(); - TopicAndPartition tp = new TopicAndPartition(ssp.getStream(), ssp.getPartition().getPartitionId()); - Long lag = latestLags.get(ssp); - LOG.trace("Latest offset of {} is {}; lag = {}", ssp, offset, lag); - if (lag != null && offset != null && lag >= 0) { - long streamEndOffset = offset.longValue() + lag.longValue(); - // update the metrics - kafkaConsumerMetrics.setHighWatermarkValue(tp, streamEndOffset); - kafkaConsumerMetrics.setLagValue(tp, lag.longValue()); - } - } - } - - @Override - public String toString() { - return String.format("consumerProxy-%s-%s", systemName, clientId); - } -} - From ee956377d9b8d4b626c6c9d463b46a863c016819 Mon Sep 17 00:00:00 2001 From: Boris S Date: Thu, 18 Oct 2018 15:45:09 -0700 Subject: [PATCH 09/18] changed to new AdminClient --- .../samza/system/kafka/KafkaSystemAdmin.java | 62 +++++++++++++++++-- .../kafka/TestKafkaSystemAdminJava.java | 15 +++-- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index d2ceafb07f..fb5324b61b 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -27,6 +27,8 @@ import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -34,10 +36,16 @@ import kafka.utils.ZkUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.DeleteTopicsResult; +import org.apache.kafka.clients.admin.DescribeTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.TopicExistsException; import org.apache.samza.Partition; import org.apache.samza.SamzaException; import org.apache.samza.config.Config; @@ -73,6 +81,7 @@ public class KafkaSystemAdmin implements ExtendedSystemAdmin { protected static final long DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS = 10000; protected static final int MAX_RETRIES_ON_EXCEPTION = 5; protected static final int DEFAULT_REPL_FACTOR = 2; + private static final int KAFKA_ADMIN_OPS_TIMEOUT_MS = 50000; // used in TestRepartitionJoinWindowApp TODO - remove SAMZA-1945 @VisibleForTesting @@ -80,7 +89,7 @@ public class KafkaSystemAdmin implements ExtendedSystemAdmin { protected final String systemName; protected final Consumer metadataConsumer; - protected final Config config; + protected final Config config; protected AdminClient adminClient = null; @@ -99,6 +108,9 @@ public class KafkaSystemAdmin implements ExtendedSystemAdmin { // used for intermediate streams protected final boolean deleteCommittedMessages; + // admin client for create/remove topics + private final org.apache.kafka.clients.admin.AdminClient newAdminClient; + private final AtomicBoolean stopped = new AtomicBoolean(false); public KafkaSystemAdmin(String systemName, Config config, Consumer metadataConsumer) { @@ -111,6 +123,10 @@ public KafkaSystemAdmin(String systemName, Config config, Consumer metadataConsu } this.metadataConsumer = metadataConsumer; + Properties props = createAdminClientProperties(); + LOG.info("New admin client with props:" + props); + newAdminClient = org.apache.kafka.clients.admin.AdminClient.create(props); + KafkaConfig kafkaConfig = new KafkaConfig(config); coordinatorStreamReplicationFactor = Integer.valueOf(kafkaConfig.getCoordinatorReplicationFactor()); coordinatorStreamProperties = KafkaSystemAdminUtilsScala.getCoordinatorTopicProperties(kafkaConfig); @@ -517,17 +533,52 @@ public Integer offsetComparator(String offset1, String offset2) { public boolean createStream(StreamSpec streamSpec) { LOG.info("Creating Kafka topic: {} on system: {}", streamSpec.getPhysicalName(), streamSpec.getSystemName()); - return KafkaSystemAdminUtilsScala.createStream(toKafkaSpec(streamSpec), getZkConnection()); + KafkaStreamSpec kSpec = toKafkaSpec(streamSpec); + String topicName = kSpec.getPhysicalName(); + + // create topic. + NewTopic newTopic = new NewTopic(topicName, kSpec.getPartitionCount(), (short) kSpec.getReplicationFactor()); + CreateTopicsResult result = newAdminClient.createTopics(ImmutableSet.of(newTopic)); + try { + result.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (Exception e) { + if (e instanceof TopicExistsException || e.getCause() instanceof TopicExistsException) { + LOG.info("Topic {} already exists.", topicName); + return false; + } + + throw new SamzaException("Create topic failed", e); + } + LOG.info("Successfully created topic {}", topicName); + return true; } @Override public boolean clearStream(StreamSpec streamSpec) { LOG.info("Creating Kafka topic: {} on system: {}", streamSpec.getPhysicalName(), streamSpec.getSystemName()); - KafkaSystemAdminUtilsScala.clearStream(streamSpec, getZkConnection()); + //KafkaSystemAdminUtilsScala.clearStream(streamSpec, getZkConnection()); + + //Map> topicsMetadata = getTopicMetadata(ImmutableSet.of(streamSpec.getPhysicalName())); + String topicName = streamSpec.getPhysicalName(); + DescribeTopicsResult dtr = newAdminClient.describeTopics(ImmutableSet.of(topicName)); + TopicDescription td = null; + try { + td = dtr.all().get().get(topicName); + System.out.println("td for {} is {}" + topicName + td); + } catch (Exception e) { + System.out.println("ERROR" + e); + } - Map> topicsMetadata = getTopicMetadata(ImmutableSet.of(streamSpec.getPhysicalName())); - return topicsMetadata.get(streamSpec.getPhysicalName()).isEmpty(); + try { + DeleteTopicsResult deleteTopicsResult = newAdminClient.deleteTopics(ImmutableSet.of(topicName)); + deleteTopicsResult.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (Exception e) { + System.out.println("Failed to delete topic {} with exception {} " + topicName + "\n" + e + "\n" + e.getCause()); + return false; + } + return true; + //return topicsMetadata.get(streamSpec.getPhysicalName()).isEmpty(); } /** @@ -629,7 +680,6 @@ protected Properties createAdminClientProperties() { } props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList); - // kafka.admin.AdminUtils requires zkConnect // this will change after we move to the new org.apache..AdminClient String zkConnect = diff --git a/samza-kafka/src/test/java/org/apache/samza/system/kafka/TestKafkaSystemAdminJava.java b/samza-kafka/src/test/java/org/apache/samza/system/kafka/TestKafkaSystemAdminJava.java index 27601b05e8..f2a6cd37b2 100644 --- a/samza-kafka/src/test/java/org/apache/samza/system/kafka/TestKafkaSystemAdminJava.java +++ b/samza-kafka/src/test/java/org/apache/samza/system/kafka/TestKafkaSystemAdminJava.java @@ -20,6 +20,7 @@ package org.apache.samza.system.kafka; import com.google.common.collect.ImmutableSet; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -221,10 +222,10 @@ public void testCreateChangelogStreamHelp(final String topic) { @Test public void testCreateStream() { StreamSpec spec = new StreamSpec("testId", "testStream", "testSystem", 8); - + KafkaSystemAdmin admin = systemAdmin(); assertTrue("createStream should return true if the stream does not exist and then is created.", - systemAdmin().createStream(spec)); - systemAdmin().validateStream(spec); + admin.createStream(spec)); + admin.validateStream(spec); assertFalse("createStream should return false if the stream already exists.", systemAdmin().createStream(spec)); } @@ -259,13 +260,15 @@ public void testValidateStreamWrongName() { systemAdmin().validateStream(spec2); } - //@Test //TODO - currently the connection to ZK fails, but since it checks for empty, the tests succeeds. SAMZA-1887 + @Test //TODO - currently the connection to ZK fails, but since it checks for empty, the tests succeeds. SAMZA-1887 public void testClearStream() { StreamSpec spec = new StreamSpec("testId", "testStreamClear", "testSystem", 8); + KafkaSystemAdmin admin = systemAdmin(); assertTrue("createStream should return true if the stream does not exist and then is created.", - systemAdmin().createStream(spec)); - assertTrue(systemAdmin().clearStream(spec)); + admin.createStream(spec)); + // validate topic exists + assertTrue(admin.clearStream(spec)); ImmutableSet topics = ImmutableSet.of(spec.getPhysicalName()); Map> metadata = systemAdmin().getTopicMetadata(topics); From 21a16a3a791f1cd095153d40db1fb0f2798914b2 Mon Sep 17 00:00:00 2001 From: Boris S Date: Tue, 23 Oct 2018 13:57:45 -0700 Subject: [PATCH 10/18] use java AdminClient to create and remove topics --- .../samza/system/kafka/KafkaSystemAdmin.java | 17 +++------------ .../kafka/TestKafkaSystemAdminJava.java | 21 +++++++++++++++---- .../system/kafka/TestKafkaSystemAdmin.scala | 2 +- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index fb5324b61b..07561ccd90 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -109,7 +109,7 @@ public class KafkaSystemAdmin implements ExtendedSystemAdmin { protected final boolean deleteCommittedMessages; // admin client for create/remove topics - private final org.apache.kafka.clients.admin.AdminClient newAdminClient; + final org.apache.kafka.clients.admin.AdminClient newAdminClient; private final AtomicBoolean stopped = new AtomicBoolean(false); @@ -557,28 +557,17 @@ public boolean createStream(StreamSpec streamSpec) { public boolean clearStream(StreamSpec streamSpec) { LOG.info("Creating Kafka topic: {} on system: {}", streamSpec.getPhysicalName(), streamSpec.getSystemName()); - //KafkaSystemAdminUtilsScala.clearStream(streamSpec, getZkConnection()); - - //Map> topicsMetadata = getTopicMetadata(ImmutableSet.of(streamSpec.getPhysicalName())); String topicName = streamSpec.getPhysicalName(); - DescribeTopicsResult dtr = newAdminClient.describeTopics(ImmutableSet.of(topicName)); - TopicDescription td = null; - try { - td = dtr.all().get().get(topicName); - System.out.println("td for {} is {}" + topicName + td); - } catch (Exception e) { - System.out.println("ERROR" + e); - } try { DeleteTopicsResult deleteTopicsResult = newAdminClient.deleteTopics(ImmutableSet.of(topicName)); deleteTopicsResult.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS); } catch (Exception e) { - System.out.println("Failed to delete topic {} with exception {} " + topicName + "\n" + e + "\n" + e.getCause()); + LOG.error("Failed to delete topic {} with exception {} " + topicName + "\n" + e + "\n" + e.getCause()); return false; } + return true; - //return topicsMetadata.get(streamSpec.getPhysicalName()).isEmpty(); } /** diff --git a/samza-kafka/src/test/java/org/apache/samza/system/kafka/TestKafkaSystemAdminJava.java b/samza-kafka/src/test/java/org/apache/samza/system/kafka/TestKafkaSystemAdminJava.java index f2a6cd37b2..c10a1f6930 100644 --- a/samza-kafka/src/test/java/org/apache/samza/system/kafka/TestKafkaSystemAdminJava.java +++ b/samza-kafka/src/test/java/org/apache/samza/system/kafka/TestKafkaSystemAdminJava.java @@ -25,6 +25,8 @@ import java.util.List; import java.util.Map; import kafka.api.TopicMetadata; +import org.apache.kafka.clients.admin.DescribeTopicsResult; +import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.PartitionInfo; import org.apache.samza.Partition; import org.apache.samza.config.ApplicationConfig; @@ -260,18 +262,29 @@ public void testValidateStreamWrongName() { systemAdmin().validateStream(spec2); } - @Test //TODO - currently the connection to ZK fails, but since it checks for empty, the tests succeeds. SAMZA-1887 + @Test public void testClearStream() { StreamSpec spec = new StreamSpec("testId", "testStreamClear", "testSystem", 8); KafkaSystemAdmin admin = systemAdmin(); + String topicName = spec.getPhysicalName(); + assertTrue("createStream should return true if the stream does not exist and then is created.", admin.createStream(spec)); // validate topic exists assertTrue(admin.clearStream(spec)); - ImmutableSet topics = ImmutableSet.of(spec.getPhysicalName()); - Map> metadata = systemAdmin().getTopicMetadata(topics); - assertTrue(metadata.get(spec.getPhysicalName()).isEmpty()); + // validate that topic was removed + DescribeTopicsResult dtr = admin.newAdminClient.describeTopics(ImmutableSet.of(topicName)); + try { + TopicDescription td = dtr.all().get().get(topicName); + Assert.fail("topic " + topicName + " should've been removed. td=" + td); + } catch (Exception e) { + if (e.getCause() instanceof org.apache.kafka.common.errors.UnknownTopicOrPartitionException) { + // expected + } else { + Assert.fail("topic " + topicName + " should've been removed. Expected UnknownTopicOrPartitionException."); + } + } } } diff --git a/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemAdmin.scala b/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemAdmin.scala index 15703635bf..d43358bcc8 100644 --- a/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemAdmin.scala +++ b/samza-kafka/src/test/scala/org/apache/samza/system/kafka/TestKafkaSystemAdmin.scala @@ -61,7 +61,7 @@ object TestKafkaSystemAdmin extends KafkaServerTestHarness { var systemAdmin: KafkaSystemAdmin = null override def generateConfigs(): Seq[KafkaConfig] = { - val props = TestUtils.createBrokerConfigs(numBrokers, zkConnect, true) + val props = TestUtils.createBrokerConfigs(numBrokers, zkConnect, enableDeleteTopic = true) props.map(KafkaConfig.fromProps) } From 8410445166eb1a4dde0673d6311dcb67b590a790 Mon Sep 17 00:00:00 2001 From: Boris S Date: Tue, 23 Oct 2018 18:10:23 -0700 Subject: [PATCH 11/18] added describe after create --- .../apache/samza/system/kafka/KafkaSystemAdmin.java | 10 +++++++++- .../checkpoint/kafka/TestKafkaCheckpointManager.scala | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index 07561ccd90..c1c9b011d3 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -550,7 +550,15 @@ public boolean createStream(StreamSpec streamSpec) { throw new SamzaException("Create topic failed", e); } LOG.info("Successfully created topic {}", topicName); - return true; + DescribeTopicsResult desc = newAdminClient.describeTopics(ImmutableSet.of(topicName)); + try { + TopicDescription td = desc.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS).get(topicName); + LOG.info("Topic {} created with {}", topicName, td); + return true; + } catch (Exception e) { + LOG.error("'Describe after create' failed for topic " + topicName, e); + return false; + } } @Override diff --git a/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala b/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala index 8d92f4d332..392670bd0f 100644 --- a/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala +++ b/samza-kafka/src/test/scala/org/apache/samza/checkpoint/kafka/TestKafkaCheckpointManager.scala @@ -77,6 +77,7 @@ class TestKafkaCheckpointManager extends KafkaServerTestHarness { kcm1.createResources kcm1.start kcm1.stop + // check that start actually creates the topic with log compaction enabled val zkClient = ZkUtils(zkConnect, 6000, 6000, JaasUtils.isZkSecurityEnabled()) val topicConfig = AdminUtils.fetchEntityConfig(zkClient, ConfigType.Topic, checkpointTopic) From e40adda405b6cb41ed3be0000e58a11825f6e496 Mon Sep 17 00:00:00 2001 From: Boris S Date: Wed, 31 Oct 2018 09:56:01 -0700 Subject: [PATCH 12/18] provide configs with the topic --- .../java/org/apache/samza/system/kafka/KafkaSystemAdmin.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index c1c9b011d3..87764fbc41 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -538,6 +538,8 @@ public boolean createStream(StreamSpec streamSpec) { // create topic. NewTopic newTopic = new NewTopic(topicName, kSpec.getPartitionCount(), (short) kSpec.getReplicationFactor()); + // specify the configs + newTopic.configs(streamSpec.getConfig()); CreateTopicsResult result = newAdminClient.createTopics(ImmutableSet.of(newTopic)); try { result.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS); From b23f31ba3237eaa9dcfcf3f37bad17421798cf3a Mon Sep 17 00:00:00 2001 From: Boris S Date: Wed, 31 Oct 2018 11:26:04 -0700 Subject: [PATCH 13/18] manually remove replication.factor form create stream configs --- .../samza/system/kafka/KafkaSystemAdmin.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index 87764fbc41..a0de0c5359 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -50,6 +50,8 @@ import org.apache.samza.SamzaException; import org.apache.samza.config.Config; import org.apache.samza.config.KafkaConfig; +import org.apache.samza.config.MapConfig; +import org.apache.samza.config.StreamConfig; import org.apache.samza.config.SystemConfig; import org.apache.samza.system.ExtendedSystemAdmin; import org.apache.samza.system.StreamSpec; @@ -532,14 +534,24 @@ public Integer offsetComparator(String offset1, String offset2) { @Override public boolean createStream(StreamSpec streamSpec) { LOG.info("Creating Kafka topic: {} on system: {}", streamSpec.getPhysicalName(), streamSpec.getSystemName()); + final String REPL_FACTOR = "replication.factor"; KafkaStreamSpec kSpec = toKafkaSpec(streamSpec); String topicName = kSpec.getPhysicalName(); // create topic. NewTopic newTopic = new NewTopic(topicName, kSpec.getPartitionCount(), (short) kSpec.getReplicationFactor()); + // specify the configs - newTopic.configs(streamSpec.getConfig()); + Map streamConfig = new HashMap(streamSpec.getConfig()); + // HACK - replication.factor is invalid config for AdminClient.createTopics + if (streamConfig.containsKey(REPL_FACTOR)) { + String repl = streamConfig.get(REPL_FACTOR); + LOG.warn("replication.factor (topic={}, repl={}) is invalid config for AdminClient.createTopics. Removing it. Using kSpec repl factor {}", + kSpec.getPhysicalName(), repl, kSpec.getReplicationFactor()); + streamConfig.remove(REPL_FACTOR); + } + newTopic.configs(new MapConfig(streamConfig)); CreateTopicsResult result = newAdminClient.createTopics(ImmutableSet.of(newTopic)); try { result.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS); From 3b30f86de871cbe092a566eb82a6bf8ca04a89e4 Mon Sep 17 00:00:00 2001 From: Boris S Date: Wed, 31 Oct 2018 11:31:39 -0700 Subject: [PATCH 14/18] remove scala create/clear streams --- .../kafka/KafkaSystemAdminUtilsScala.scala | 93 +------------------ 1 file changed, 4 insertions(+), 89 deletions(-) diff --git a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemAdminUtilsScala.scala b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemAdminUtilsScala.scala index 6ff2b505d2..89ba1bbddc 100644 --- a/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemAdminUtilsScala.scala +++ b/samza-kafka/src/main/scala/org/apache/samza/system/kafka/KafkaSystemAdminUtilsScala.scala @@ -22,21 +22,18 @@ package org.apache.samza.system.kafka import java.util import java.util.Properties -import kafka.admin.{AdminClient, AdminUtils} -import kafka.utils.{Logging, ZkUtils} +import kafka.admin.AdminClient +import kafka.utils.Logging import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.errors.TopicExistsException import org.apache.samza.config.ApplicationConfig.ApplicationMode import org.apache.samza.config.{ApplicationConfig, Config, KafkaConfig, StreamConfig} import org.apache.samza.system.SystemStreamMetadata.SystemStreamPartitionMetadata -import org.apache.samza.system.{StreamSpec, SystemStreamMetadata, SystemStreamPartition} -import org.apache.samza.util.ExponentialSleepStrategy -import org.slf4j.{Logger, LoggerFactory} +import org.apache.samza.system.{SystemStreamMetadata, SystemStreamPartition} import scala.collection.JavaConverters._ /** - * A helper class that is used to construct the changelog stream specific information + * A helper class for KafkaSystemAdmin * * @param replicationFactor The number of replicas for the changelog stream * @param kafkaProps The kafka specific properties that need to be used for changelog stream creation @@ -47,88 +44,6 @@ case class ChangelogInfo(var replicationFactor: Int, var kafkaProps: Properties) // TODO move to org.apache.kafka.clients.admin.AdminClien from the kafka.admin.AdminClient object KafkaSystemAdminUtilsScala extends Logging { - val CLEAR_STREAM_RETRIES = 3 - val CREATE_STREAM_RETRIES = 10 - - /** - * @inheritdoc - * - * Delete a stream in Kafka. Deleting topics works only when the broker is configured with "delete.topic.enable=true". - * Otherwise it's a no-op. - */ - def clearStream(spec: StreamSpec, connectZk: java.util.function.Supplier[ZkUtils]): Unit = { - info("Deleting topic %s for system %s" format(spec.getPhysicalName, spec.getSystemName)) - val kSpec = KafkaStreamSpec.fromSpec(spec) - var retries = CLEAR_STREAM_RETRIES - new ExponentialSleepStrategy().run( - loop => { - val zkClient = connectZk.get() - try { - AdminUtils.deleteTopic( - zkClient, - kSpec.getPhysicalName) - } finally { - zkClient.close - } - - loop.done - }, - - (exception, loop) => { - if (retries > 0) { - warn("Exception while trying to delete topic %s. Retrying." format (spec.getPhysicalName), exception) - retries -= 1 - } else { - warn("Fail to delete topic %s." format (spec.getPhysicalName), exception) - loop.done - throw exception - } - }) - } - - - def createStream(kSpec: KafkaStreamSpec, connectZk: java.util.function.Supplier[ZkUtils]): Boolean = { - info("Creating topic %s for system %s" format(kSpec.getPhysicalName, kSpec.getSystemName)) - var streamCreated = false - var retries = CREATE_STREAM_RETRIES - - new ExponentialSleepStrategy(initialDelayMs = 500).run( - loop => { - val zkClient = connectZk.get() - try { - AdminUtils.createTopic( - zkClient, - kSpec.getPhysicalName, - kSpec.getPartitionCount, - kSpec.getReplicationFactor, - kSpec.getProperties) - } finally { - zkClient.close - } - - streamCreated = true - loop.done - }, - - (exception, loop) => { - exception match { - case e: TopicExistsException => - streamCreated = false - loop.done - case e: Exception => - if (retries > 0) { - warn("Failed to create topic %s. Retrying." format (kSpec.getPhysicalName), exception) - retries -= 1 - } else { - error("Failed to create topic %s. Bailing out." format (kSpec.getPhysicalName), exception) - throw exception - } - } - }) - - streamCreated - } - /** * A helper method that takes oldest, newest, and upcoming offsets for each * system stream partition, and creates a single map from stream name to From d57a878da0e4fd117ec748a7e256aba5187a2640 Mon Sep 17 00:00:00 2001 From: Boris S Date: Wed, 31 Oct 2018 11:45:49 -0700 Subject: [PATCH 15/18] cleanup --- .../samza/system/kafka/KafkaSystemAdmin.java | 32 ++++++++++--------- .../kafka/TestKafkaSystemAdminJava.java | 9 ++---- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index a0de0c5359..e7deb4f81c 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -27,15 +27,14 @@ import java.util.Map; import java.util.Properties; import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import java.util.stream.Collectors; -import kafka.admin.AdminClient; import kafka.utils.ZkUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; +import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.CreateTopicsResult; import org.apache.kafka.clients.admin.DeleteTopicsResult; import org.apache.kafka.clients.admin.DescribeTopicsResult; @@ -51,7 +50,6 @@ import org.apache.samza.config.Config; import org.apache.samza.config.KafkaConfig; import org.apache.samza.config.MapConfig; -import org.apache.samza.config.StreamConfig; import org.apache.samza.config.SystemConfig; import org.apache.samza.system.ExtendedSystemAdmin; import org.apache.samza.system.StreamSpec; @@ -93,7 +91,7 @@ public class KafkaSystemAdmin implements ExtendedSystemAdmin { protected final Consumer metadataConsumer; protected final Config config; - protected AdminClient adminClient = null; + protected kafka.admin.AdminClient adminClientForDelete = null; // Custom properties to create a new coordinator stream. private final Properties coordinatorStreamProperties; @@ -111,7 +109,7 @@ public class KafkaSystemAdmin implements ExtendedSystemAdmin { protected final boolean deleteCommittedMessages; // admin client for create/remove topics - final org.apache.kafka.clients.admin.AdminClient newAdminClient; + final AdminClient adminClient; private final AtomicBoolean stopped = new AtomicBoolean(false); @@ -127,7 +125,7 @@ public KafkaSystemAdmin(String systemName, Config config, Consumer metadataConsu Properties props = createAdminClientProperties(); LOG.info("New admin client with props:" + props); - newAdminClient = org.apache.kafka.clients.admin.AdminClient.create(props); + adminClient = AdminClient.create(props); KafkaConfig kafkaConfig = new KafkaConfig(config); coordinatorStreamReplicationFactor = Integer.valueOf(kafkaConfig.getCoordinatorReplicationFactor()); @@ -185,13 +183,17 @@ public void stop() { LOG.warn("metadataConsumer.close for system " + systemName + " failed with exception.", e); } } - if (adminClient != null) { + if (adminClientForDelete != null) { try { - adminClient.close(); + adminClientForDelete.close(); } catch (Exception e) { - LOG.warn("adminClient.close for system " + systemName + " failed with exception.", e); + LOG.warn("AdminClient.close() for system " + systemName + " failed with exception.", e); } } + + if (adminClient != null) { + adminClient.close(); + } } /** @@ -552,7 +554,7 @@ public boolean createStream(StreamSpec streamSpec) { streamConfig.remove(REPL_FACTOR); } newTopic.configs(new MapConfig(streamConfig)); - CreateTopicsResult result = newAdminClient.createTopics(ImmutableSet.of(newTopic)); + CreateTopicsResult result = adminClient.createTopics(ImmutableSet.of(newTopic)); try { result.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS); } catch (Exception e) { @@ -564,7 +566,7 @@ public boolean createStream(StreamSpec streamSpec) { throw new SamzaException("Create topic failed", e); } LOG.info("Successfully created topic {}", topicName); - DescribeTopicsResult desc = newAdminClient.describeTopics(ImmutableSet.of(topicName)); + DescribeTopicsResult desc = adminClient.describeTopics(ImmutableSet.of(topicName)); try { TopicDescription td = desc.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS).get(topicName); LOG.info("Topic {} created with {}", topicName, td); @@ -582,7 +584,7 @@ public boolean clearStream(StreamSpec streamSpec) { String topicName = streamSpec.getPhysicalName(); try { - DeleteTopicsResult deleteTopicsResult = newAdminClient.deleteTopics(ImmutableSet.of(topicName)); + DeleteTopicsResult deleteTopicsResult = adminClient.deleteTopics(ImmutableSet.of(topicName)); deleteTopicsResult.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS); } catch (Exception e) { LOG.error("Failed to delete topic {} with exception {} " + topicName + "\n" + e + "\n" + e.getCause()); @@ -663,10 +665,10 @@ Map> getTopicMetadata(Set topics) { @Override public void deleteMessages(Map offsets) { if (deleteCommittedMessages) { - if (adminClient == null) { - adminClient = AdminClient.create(createAdminClientProperties()); + if (adminClientForDelete == null) { + adminClientForDelete = AdminClient.create(createAdminClientProperties()); } - KafkaSystemAdminUtilsScala.deleteMessages(adminClient, offsets); + KafkaSystemAdminUtilsScala.deleteMessages(adminClientForDelete, offsets); deleteMessageCalled = true; } } diff --git a/samza-kafka/src/test/java/org/apache/samza/system/kafka/TestKafkaSystemAdminJava.java b/samza-kafka/src/test/java/org/apache/samza/system/kafka/TestKafkaSystemAdminJava.java index c10a1f6930..1d305fb6a0 100644 --- a/samza-kafka/src/test/java/org/apache/samza/system/kafka/TestKafkaSystemAdminJava.java +++ b/samza-kafka/src/test/java/org/apache/samza/system/kafka/TestKafkaSystemAdminJava.java @@ -20,14 +20,10 @@ package org.apache.samza.system.kafka; import com.google.common.collect.ImmutableSet; -import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; -import kafka.api.TopicMetadata; import org.apache.kafka.clients.admin.DescribeTopicsResult; import org.apache.kafka.clients.admin.TopicDescription; -import org.apache.kafka.common.PartitionInfo; import org.apache.samza.Partition; import org.apache.samza.config.ApplicationConfig; import org.apache.samza.config.Config; @@ -269,13 +265,12 @@ public void testClearStream() { KafkaSystemAdmin admin = systemAdmin(); String topicName = spec.getPhysicalName(); - assertTrue("createStream should return true if the stream does not exist and then is created.", - admin.createStream(spec)); + assertTrue("createStream should return true if the stream does not exist and then is created.", admin.createStream(spec)); // validate topic exists assertTrue(admin.clearStream(spec)); // validate that topic was removed - DescribeTopicsResult dtr = admin.newAdminClient.describeTopics(ImmutableSet.of(topicName)); + DescribeTopicsResult dtr = admin.adminClient.describeTopics(ImmutableSet.of(topicName)); try { TopicDescription td = dtr.all().get().get(topicName); Assert.fail("topic " + topicName + " should've been removed. td=" + td); From 1fe59fd2769d4097b1d3c42941fbf0e8ee9a5066 Mon Sep 17 00:00:00 2001 From: Boris S Date: Thu, 1 Nov 2018 16:34:00 -0700 Subject: [PATCH 16/18] use the correct AdminClient --- .../java/org/apache/samza/system/kafka/KafkaSystemAdmin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index e7deb4f81c..ad8e48f76a 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -666,7 +666,7 @@ Map> getTopicMetadata(Set topics) { public void deleteMessages(Map offsets) { if (deleteCommittedMessages) { if (adminClientForDelete == null) { - adminClientForDelete = AdminClient.create(createAdminClientProperties()); + adminClientForDelete = kafka.admin.AdminClient.create(createAdminClientProperties()); } KafkaSystemAdminUtilsScala.deleteMessages(adminClientForDelete, offsets); deleteMessageCalled = true; From bdc69ae699752cce5e00c09b7ad9cc9a575ac0ad Mon Sep 17 00:00:00 2001 From: svenkata Date: Fri, 2 Nov 2018 14:06:43 -0700 Subject: [PATCH 17/18] Update samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java Co-Authored-By: sborya --- .../java/org/apache/samza/system/kafka/KafkaSystemAdmin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index ad8e48f76a..4fcf7d24ec 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -563,7 +563,7 @@ public boolean createStream(StreamSpec streamSpec) { return false; } - throw new SamzaException("Create topic failed", e); + throw new SamzaException(String.format("Creation of topic: %s failed.", topicName), e); } LOG.info("Successfully created topic {}", topicName); DescribeTopicsResult desc = adminClient.describeTopics(ImmutableSet.of(topicName)); From 3b5b3dca41bb297526eea71799c460edb8821c1e Mon Sep 17 00:00:00 2001 From: Boris S Date: Fri, 2 Nov 2018 14:11:26 -0700 Subject: [PATCH 18/18] review --- .../apache/samza/system/kafka/KafkaSystemAdmin.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java index 17ab0ff506..596b07a9bc 100644 --- a/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java +++ b/samza-kafka/src/main/java/org/apache/samza/system/kafka/KafkaSystemAdmin.java @@ -187,7 +187,7 @@ public void stop() { try { adminClientForDelete.close(); } catch (Exception e) { - LOG.warn("AdminClient.close() for system " + systemName + " failed with exception.", e); + LOG.warn("AdminClient.close() for system {} failed with exception {}.", systemName, e); } } @@ -514,8 +514,8 @@ public boolean createStream(StreamSpec streamSpec) { // HACK - replication.factor is invalid config for AdminClient.createTopics if (streamConfig.containsKey(REPL_FACTOR)) { String repl = streamConfig.get(REPL_FACTOR); - LOG.warn("replication.factor (topic={}, repl={}) is invalid config for AdminClient.createTopics. Removing it. Using kSpec repl factor {}", - kSpec.getPhysicalName(), repl, kSpec.getReplicationFactor()); + LOG.warn("Configuration {}={} for topic={} is invalid. Using kSpec repl factor {}", + REPL_FACTOR, repl, kSpec.getPhysicalName(), kSpec.getReplicationFactor()); streamConfig.remove(REPL_FACTOR); } newTopic.configs(new MapConfig(streamConfig)); @@ -528,7 +528,7 @@ public boolean createStream(StreamSpec streamSpec) { return false; } - throw new SamzaException(String.format("Creation of topic: %s failed.", topicName), e); + throw new SamzaException(String.format("Creation of topic %s failed.", topicName), e); } LOG.info("Successfully created topic {}", topicName); DescribeTopicsResult desc = adminClient.describeTopics(ImmutableSet.of(topicName)); @@ -552,7 +552,7 @@ public boolean clearStream(StreamSpec streamSpec) { DeleteTopicsResult deleteTopicsResult = adminClient.deleteTopics(ImmutableSet.of(topicName)); deleteTopicsResult.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS); } catch (Exception e) { - LOG.error("Failed to delete topic {} with exception {} " + topicName + "\n" + e + "\n" + e.getCause()); + LOG.error("Failed to delete topic {} with exception {}.", topicName, e); return false; }