From 71a4b4798cf2468612476a839d78d72f4b571612 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Fri, 4 Jun 2021 11:09:05 -0700 Subject: [PATCH 01/44] Replaces thread/pool with concurrent ruby gem - uses a well supported and updated gem - this should be more predictable in behaviour and not crash in a few hours --- .../fluent-plugin-jfrog-siem.gemspec | 4 ++-- .../lib/fluent/plugin/in_jfrog_siem.rb | 15 +++++---------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec b/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec index dd12bee..28d999d 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec @@ -24,7 +24,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "rake", "~> 12.0" spec.add_development_dependency "test-unit", "~> 3.0" spec.add_development_dependency "rest-client", "~> 2.0" - spec.add_development_dependency "thread", "~> 0.2.2" - spec.add_runtime_dependency "thread", "~> 0.2.2" + spec.add_development_dependency "concurrent-ruby", "~> 1.1.8" + spec.add_runtime_dependency "concurrent-ruby", "~> 1.1.8" spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"] end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 1589406..d7b1c5c 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -14,7 +14,7 @@ # limitations under the License. require "fluent/plugin/input" require "rest-client" -require "thread/pool" +require 'concurrent' require "json" require "date" require "uri" @@ -159,15 +159,10 @@ def run end end - # iterate over url array adding to thread pool each url. - # limit max workers to thread count to prevent overloading xray. - thread_pool = Thread.pool(thread_count) - thread_pool.process { - for xray_violation_url in xray_violation_urls_list do - pull_violation_details(xray_violation_url) - end - } - thread_pool.shutdown + xray_violation_urls_list.map do |xv_url| + Concurrent::Promises.future(xv_url)) { |xv| pull_violation_details xv } + end + xray_violation_urls_list.value!.map(&:value!) rescue $! # reduce left violations by jump size (not all batches have full item count??) left_violations = left_violations - @batch_size From 923fd069ee511301604190429791dc2eb9d6ecb1 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Fri, 4 Jun 2021 11:10:03 -0700 Subject: [PATCH 02/44] Adds missing Gemfile.lock - https://dragonastronauts.com/2019/08/14/should-you-add-gemfile-lock-to-git/ --- .../fluent-plugin-jfrog-siem/Gemfile.lock | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock new file mode 100644 index 0000000..692c580 --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock @@ -0,0 +1,71 @@ +PATH + remote: . + specs: + fluent-plugin-jfrog-siem (0.1.8) + concurrent-ruby (~> 1.1.8) + fluentd (>= 0.14.10, < 2) + +GEM + remote: https://rubygems.org/ + specs: + concurrent-ruby (1.1.8) + cool.io (1.7.1) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) + fluentd (1.13.0) + bundler + cool.io (>= 1.4.5, < 2.0.0) + http_parser.rb (>= 0.5.1, < 0.7.0) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.2.2, < 3.0.0) + sigdump (~> 0.2.2) + strptime (>= 0.2.2, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (>= 1.4.2, < 1.8.0) + yajl-ruby (~> 1.0) + http-accept (1.7.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http_parser.rb (0.6.0) + mime-types (3.3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2021.0225) + msgpack (1.4.2) + netrc (0.11.0) + power_assert (2.0.0) + rake (12.3.3) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + serverengine (2.2.4) + sigdump (~> 0.2.2) + sigdump (0.2.4) + strptime (0.2.5) + test-unit (3.4.2) + power_assert + tzinfo (2.0.4) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2021.1) + tzinfo (>= 1.0.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.7) + webrick (1.7.0) + yajl-ruby (1.4.1) + +PLATFORMS + x86_64-darwin-20 + +DEPENDENCIES + bundler (~> 2.0) + concurrent-ruby (~> 1.1.8) + fluent-plugin-jfrog-siem! + rake (~> 12.0) + rest-client (~> 2.0) + test-unit (~> 3.0) + +BUNDLED WITH + 2.2.3 From a4db7568bfffccd09e4c8b8a32865a7e7dc68675 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Fri, 4 Jun 2021 11:45:21 -0700 Subject: [PATCH 03/44] Rescues error and prints message to console - does not throw exception so the plugin continues to run inspite of errors in specific urls --- .../lib/fluent/plugin/in_jfrog_siem.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index d7b1c5c..d0565d5 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -162,7 +162,12 @@ def run xray_violation_urls_list.map do |xv_url| Concurrent::Promises.future(xv_url)) { |xv| pull_violation_details xv } end - xray_violation_urls_list.value!.map(&:value!) rescue $! + + begin + xray_violation_urls_list.value!.map(&:value!) + rescue => e + puts "Failed to pull violation details due to #{e}" + end # reduce left violations by jump size (not all batches have full item count??) left_violations = left_violations - @batch_size From 6e80eb619dfff10a511f612a2df545a50d51a5ee Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Tue, 8 Jun 2021 16:23:18 -0700 Subject: [PATCH 04/44] Converts code to use concurrent mechanisms from concurrent-ruby gem - uses TimerTask to pull violations periodically instead of a while loop - uses futures to handle violation details instead of threadpools - uses concurrent-array as a channel to 'Do not communicate by sharing memory; instead, share memory by communicating' - since concurrent-array is thread-safe --- .../lib/fluent/plugin/in_jfrog_siem.rb | 163 ++++++++++-------- 1 file changed, 90 insertions(+), 73 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index d0565d5..c0523f2 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -95,7 +95,7 @@ def shutdown def run call_home(@jpd_url) - # runs the violation pull + last_created_date_string = get_last_item_create_date() begin last_created_date = DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -106,81 +106,98 @@ def run left_violations=0 waiting_for_violations = false xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": offset_count } } - - while true - # Grab the batch of records - resp=get_xray_violations(xray_json, @jpd_url) - number_of_violations = JSON.parse(resp)['total_violations'] - if left_violations <= 0 - left_violations = number_of_violations - end - - xray_violation_urls_list = [] - for index in 0..JSON.parse(resp)['violations'].length-1 do - # Get the violation - item = JSON.parse(resp)['violations'][index] - - # Get the created date and check if we should skip (already processed) or process this record. - created_date_string = item['created'] - created_date = DateTime.parse(created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ") - - # Determine if we need to persist this record or not - persistItem = true - if waiting_for_violations - if created_date <= last_created_date - # "not persisting it - waiting for violations" - persistItem = false - end - else - if created_date < last_created_date - # "persisting everything" - persistItem = true - end - end - - # Publish the record to fluentd - if persistItem - - now = Fluent::Engine.now - router.emit(@tag, now, item) - - # write to the pos_file created_date_string - open(@pos_file, 'a') do |f| - f << "#{created_date_string}\n" - end - - # Mark this as the last record successfully processed - last_created_date_string = created_date_string - last_created_date = created_date - - # Grab violation detail url and add to url list to process w/ thread pool - xray_violation_details_url=item['violation_details_url'] - xray_violation_urls_list.append(xray_violation_details_url) - end - end - - xray_violation_urls_list.map do |xv_url| - Concurrent::Promises.future(xv_url)) { |xv| pull_violation_details xv } - end - - begin - xray_violation_urls_list.value!.map(&:value!) - rescue => e - puts "Failed to pull violation details due to #{e}" + + # Channel is still a concurrent-ruby-edge feature but concurrent::array is threadsafe so using that instead. + violations_channel = Concurrent::Array.new[] + + timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 5) do + violations = JSON.parse(get_xray_violations(xray_json, @jpd_url)) + violations.each do |v| + violations_channel << v end + end + timer_task.execute - # reduce left violations by jump size (not all batches have full item count??) - left_violations = left_violations - @batch_size - if left_violations <= 0 - waiting_for_violations = true - sleep(@wait_interval) - else - # Grab the next record to process for the violation details url - waiting_for_violations = false - offset_count = offset_count + 1 - xray_json={"filters": { "created_from": last_created_date_string }, "pagination": {"order_by": "created","limit": @batch_size , "offset": offset_count } } - end + violations_channel.each do |v| + Concurrent::Promises.future(v)) { |v| pull_violation_details v['violation_details_url'] } end + + # Need to add persistItem logic based on created_date + + # while true + # # Grab the batch of records + # resp=get_xray_violations(xray_json, @jpd_url) + # number_of_violations = JSON.parse(resp)['total_violations'] + # if left_violations <= 0 + # left_violations = number_of_violations + # end + + # xray_violation_urls_list = [] + # for index in 0..JSON.parse(resp)['violations'].length-1 do + # # Get the violation + # item = JSON.parse(resp)['violations'][index] + + # # Get the created date and check if we should skip (already processed) or process this record. + # created_date_string = item['created'] + # created_date = DateTime.parse(created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ") + + # # Determine if we need to persist this record or not + # persistItem = true + # if waiting_for_violations + # if created_date <= last_created_date + # # "not persisting it - waiting for violations" + # persistItem = false + # end + # else + # if created_date < last_created_date + # # "persisting everything" + # persistItem = true + # end + # end + + # # Publish the record to fluentd + # if persistItem + + # now = Fluent::Engine.now + # router.emit(@tag, now, item) + + # # write to the pos_file created_date_string + # open(@pos_file, 'a') do |f| + # f << "#{created_date_string}\n" + # end + + # # Mark this as the last record successfully processed + # last_created_date_string = created_date_string + # last_created_date = created_date + + # # Grab violation detail url and add to url list to process w/ thread pool + # xray_violation_details_url=item['violation_details_url'] + # xray_violation_urls_list.append(xray_violation_details_url) + # end + # end + + # xray_violation_urls_list.map do |xv_url| + # Concurrent::Promises.future(xv_url)) { |xv| pull_violation_details xv } + # end + + # begin + # xray_violation_urls_list.value!.map(&:value!) + # rescue => e + # puts "Failed to pull violation details due to #{e}" + # end + + # # reduce left violations by jump size (not all batches have full item count??) + # left_violations = left_violations - @batch_size + # if left_violations <= 0 + # waiting_for_violations = true + # sleep(@wait_interval) + # else + # # Grab the next record to process for the violation details url + # waiting_for_violations = false + # offset_count = offset_count + 1 + # xray_json={"filters": { "created_from": last_created_date_string }, "pagination": {"order_by": "created","limit": @batch_size , "offset": offset_count } } + # end + # end end From 54511c1c60650efb40f364c10f8b2c71a5c32abc Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Wed, 9 Jun 2021 21:22:27 -0700 Subject: [PATCH 05/44] Waits for array to be populated before processing details - since it is not blocking like channels would be. --- .../lib/fluent/plugin/in_jfrog_siem.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index c0523f2..cb415ae 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -118,9 +118,12 @@ def run end timer_task.execute - violations_channel.each do |v| - Concurrent::Promises.future(v)) { |v| pull_violation_details v['violation_details_url'] } + details_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 5) do + violations_channel.each do |v| + Concurrent::Promises.future(v)) { |v| pull_violation_details v['violation_details_url'] } + end end + details_task.execute # Need to add persistItem logic based on created_date From 2edda7e8e12ddb1c1fb34ffef8d213c20de96187 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Wed, 16 Jun 2021 15:08:07 -0700 Subject: [PATCH 06/44] Uses channels from 'concurrent-edge' gem - since they are blocking and make the timertask redundant - adds a catch block to catch failures from tests - puts errors onto stdout for any rescue blogs - to help with debugging. --- .../fluent-plugin-jfrog-siem/Gemfile.lock | 4 +++ .../fluent-plugin-jfrog-siem.gemspec | 3 ++ .../lib/fluent/plugin/in_jfrog_siem.rb | 32 +++++++++++-------- .../test/plugin/test_in_jfrog_siem.rb | 6 +++- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock index 692c580..51c1536 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock @@ -3,12 +3,15 @@ PATH specs: fluent-plugin-jfrog-siem (0.1.8) concurrent-ruby (~> 1.1.8) + concurrent-ruby-edge fluentd (>= 0.14.10, < 2) GEM remote: https://rubygems.org/ specs: concurrent-ruby (1.1.8) + concurrent-ruby-edge (0.6.0) + concurrent-ruby (~> 1.1.6) cool.io (1.7.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) @@ -62,6 +65,7 @@ PLATFORMS DEPENDENCIES bundler (~> 2.0) concurrent-ruby (~> 1.1.8) + concurrent-ruby-edge fluent-plugin-jfrog-siem! rake (~> 12.0) rest-client (~> 2.0) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec b/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec index 28d999d..00a3a9f 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec @@ -25,6 +25,9 @@ Gem::Specification.new do |spec| spec.add_development_dependency "test-unit", "~> 3.0" spec.add_development_dependency "rest-client", "~> 2.0" spec.add_development_dependency "concurrent-ruby", "~> 1.1.8" + spec.add_development_dependency "concurrent-ruby-edge", '>= 0' + spec.add_runtime_dependency "concurrent-ruby", "~> 1.1.8" + spec.add_runtime_dependency "concurrent-ruby-edge", '>= 0' spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"] end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index cb415ae..6f1509d 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -15,6 +15,7 @@ require "fluent/plugin/input" require "rest-client" require 'concurrent' +require 'concurrent-edge' require "json" require "date" require "uri" @@ -108,23 +109,21 @@ def run xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": offset_count } } # Channel is still a concurrent-ruby-edge feature but concurrent::array is threadsafe so using that instead. - violations_channel = Concurrent::Array.new[] - - timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 5) do - violations = JSON.parse(get_xray_violations(xray_json, @jpd_url)) - violations.each do |v| + violations_channel = Concurrent::Channel.new(capacity: 100) + timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do + resp = JSON.parse(get_xray_violations(xray_json, @jpd_url)) + puts "Violations count is #{resp['total_violations']}" + resp['violations'].each do |v| violations_channel << v end end timer_task.execute - details_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 5) do - violations_channel.each do |v| - Concurrent::Promises.future(v)) { |v| pull_violation_details v['violation_details_url'] } - end + violations_channel.each do |v| + puts "Collecting violation details for #{v['infected_components']}: #{v['watch_name']} : #{v['issue_id']}" + Concurrent::Promises.future(v) { |v| pull_violation_details(v['violation_details_url'])} end - details_task.execute - + sleep 100 # Need to add persistItem logic based on created_date # while true @@ -239,6 +238,7 @@ def get_xray_violations_detail(xray_violation_detail_url) when 200 return response.to_str else + puts "error: #{response.to_json}" raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." end end @@ -247,6 +247,7 @@ def get_xray_violations_detail(xray_violation_detail_url) # queries the xray API for violations based upon the input json def get_xray_violations(xray_json, jpd_url) + puts "jpd_url for #{jpd_url}" response = RestClient::Request.new( :method => :post, :url => jpd_url + "/xray/api/v1/violations", @@ -259,7 +260,8 @@ def get_xray_violations(xray_json, jpd_url) when 200 return response.to_str else - raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." + puts "error: #{response.to_json}" + raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations. #{response.to_json}" end end end @@ -327,12 +329,14 @@ def data_normalization(detailResp) def pull_violation_details(xray_violation_detail_url) begin + puts "Pulling violation details for #{xray_violation_detail_url}" detailResp=get_xray_violations_detail(xray_violation_detail_url) time = Fluent::Engine.now detailResp_json = data_normalization(detailResp) router.emit(@tag, time, detailResp_json) - rescue - raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}" + rescue => e + puts "error: #{e}" + raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" end end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb index 5ca271b..c77f319 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb @@ -28,7 +28,11 @@ def create_driver(conf = CONFIG) sub_test_case 'Testing' do test 'Testing plugin in_jfrog_siem' do d = create_driver(CONFIG) - d.run + begin + d.run + rescue => e + raise "Test failed due to #{e}" + end end end end From 5123a7a0518f3081aaa4573d98c21fed19e724b4 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Wed, 16 Jun 2021 21:32:45 -0700 Subject: [PATCH 07/44] Logs the created date to the pos file --- .../lib/fluent/plugin/in_jfrog_siem.rb | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 6f1509d..79a92c6 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -108,7 +108,6 @@ def run waiting_for_violations = false xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": offset_count } } - # Channel is still a concurrent-ruby-edge feature but concurrent::array is threadsafe so using that instead. violations_channel = Concurrent::Channel.new(capacity: 100) timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do resp = JSON.parse(get_xray_violations(xray_json, @jpd_url)) @@ -121,7 +120,17 @@ def run violations_channel.each do |v| puts "Collecting violation details for #{v['infected_components']}: #{v['watch_name']} : #{v['issue_id']}" - Concurrent::Promises.future(v) { |v| pull_violation_details(v['violation_details_url'])} + Concurrent::Promises.future(v) do |v| + puts "In the future for #{v}" + created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + # puts created_date + open(@pos_file, 'a') do |f| + puts created_date + f.puts created_date + end + + pull_violation_details(v['violation_details_url']) + end end sleep 100 # Need to add persistItem logic based on created_date From 580f3a5553dab7bf13abdd26df70d5b5e8f8999f Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Thu, 17 Jun 2021 11:00:37 -0700 Subject: [PATCH 08/44] Adds violation ids to help with debugging to the pos file - increments page number once the requested page is processed --- .../lib/fluent/plugin/in_jfrog_siem.rb | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 79a92c6..ba75dca 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -103,30 +103,28 @@ def run rescue last_created_date = DateTime.parse("1970-01-01T00:00:00Z").strftime("%Y-%m-%dT%H:%M:%SZ") end - offset_count=1 + page_number=1 left_violations=0 waiting_for_violations = false - xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": offset_count } } violations_channel = Concurrent::Channel.new(capacity: 100) timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do + xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } resp = JSON.parse(get_xray_violations(xray_json, @jpd_url)) puts "Violations count is #{resp['total_violations']}" resp['violations'].each do |v| violations_channel << v end + page_number += 1 end timer_task.execute violations_channel.each do |v| - puts "Collecting violation details for #{v['infected_components']}: #{v['watch_name']} : #{v['issue_id']}" Concurrent::Promises.future(v) do |v| - puts "In the future for #{v}" - created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") - # puts created_date + puts "In future: Collecting violation details for #{v['infected_components']}: #{v['watch_name']} : #{v['issue_id']}" open(@pos_file, 'a') do |f| - puts created_date - f.puts created_date + created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + f.puts [created_date, v['watch_name'], v['issue_id']].join(',') end pull_violation_details(v['violation_details_url']) From 41b2b916694b36f47220553c6004078e5c50bc33 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Mon, 21 Jun 2021 15:36:52 -0700 Subject: [PATCH 09/44] Attempts to add rspec and specs --- .gitignore | 3 +- .../input/fluent-plugin-jfrog-siem/.rspec | 1 + .../fluent-plugin-jfrog-siem/Gemfile.lock | 15 + .../fluent-plugin-jfrog-siem.gemspec | 1 + .../lib/fluent/plugin/in_jfrog_siem.rb | 78 ++-- .../lib/fluent/plugin/violations.json | 380 ++++++++++++++++++ .../lib/fluent/plugin/xray.rb | 68 ++++ .../spec/spec_helper.rb | 111 +++++ .../spec/xray_spec.rb | 47 +++ .../fluent-plugin-jfrog-siem/test/helper.rb | 1 + .../test/plugin/test_in_jfrog_siem.rb | 18 +- 11 files changed, 670 insertions(+), 53 deletions(-) create mode 100644 fluentd/plugins/input/fluent-plugin-jfrog-siem/.rspec create mode 100644 fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/violations.json create mode 100644 fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb create mode 100644 fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/spec_helper.rb create mode 100644 fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb diff --git a/.gitignore b/.gitignore index 9f6676b..0f98557 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ .tox ./idea .idea/ -.DS_Store \ No newline at end of file +.DS_Store +**/test_pos.txt \ No newline at end of file diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/.rspec b/fluentd/plugins/input/fluent-plugin-jfrog-siem/.rspec new file mode 100644 index 0000000..c99d2e7 --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/.rspec @@ -0,0 +1 @@ +--require spec_helper diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock index 51c1536..efea5f2 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock @@ -13,6 +13,7 @@ GEM concurrent-ruby-edge (0.6.0) concurrent-ruby (~> 1.1.6) cool.io (1.7.1) + diff-lcs (1.4.4) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) fluentd (1.13.0) @@ -43,6 +44,19 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) + rspec (3.10.0) + rspec-core (~> 3.10.0) + rspec-expectations (~> 3.10.0) + rspec-mocks (~> 3.10.0) + rspec-core (3.10.1) + rspec-support (~> 3.10.0) + rspec-expectations (3.10.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.10.0) + rspec-mocks (3.10.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.10.0) + rspec-support (3.10.2) serverengine (2.2.4) sigdump (~> 0.2.2) sigdump (0.2.4) @@ -69,6 +83,7 @@ DEPENDENCIES fluent-plugin-jfrog-siem! rake (~> 12.0) rest-client (~> 2.0) + rspec (~> 3.10.0) test-unit (~> 3.0) BUNDLED WITH diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec b/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec index 00a3a9f..7ef8b36 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec @@ -26,6 +26,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "rest-client", "~> 2.0" spec.add_development_dependency "concurrent-ruby", "~> 1.1.8" spec.add_development_dependency "concurrent-ruby-edge", '>= 0' + spec.add_development_dependency 'rspec', '~> 3.10.0' spec.add_runtime_dependency "concurrent-ruby", "~> 1.1.8" spec.add_runtime_dependency "concurrent-ruby-edge", '>= 0' diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index ba75dca..5b336ab 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -14,11 +14,9 @@ # limitations under the License. require "fluent/plugin/input" require "rest-client" -require 'concurrent' -require 'concurrent-edge' -require "json" require "date" require "uri" +require 'xray' module Fluent module Plugin @@ -103,33 +101,37 @@ def run rescue last_created_date = DateTime.parse("1970-01-01T00:00:00Z").strftime("%Y-%m-%dT%H:%M:%SZ") end - page_number=1 - left_violations=0 - waiting_for_violations = false + date_since = last_created_date - violations_channel = Concurrent::Channel.new(capacity: 100) - timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do - xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } - resp = JSON.parse(get_xray_violations(xray_json, @jpd_url)) - puts "Violations count is #{resp['total_violations']}" - resp['violations'].each do |v| - violations_channel << v - end - page_number += 1 - end - timer_task.execute - - violations_channel.each do |v| - Concurrent::Promises.future(v) do |v| - puts "In future: Collecting violation details for #{v['infected_components']}: #{v['watch_name']} : #{v['issue_id']}" - open(@pos_file, 'a') do |f| - created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") - f.puts [created_date, v['watch_name'], v['issue_id']].join(',') - end + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) + violations_channel = xray.violations(date_since) - pull_violation_details(v['violation_details_url']) - end - end + xray.violation_details(violations_channel) + + + # violations_channel = Concurrent::Channel.new(capacity: 100) + # timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do + # xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } + # resp = JSON.parse(get_xray_violations(xray_json, @jpd_url)) + # puts "Violations count is #{resp['total_violations']}" + # resp['violations'].each do |v| + # violations_channel << v + # end + # page_number += 1 + # end + # timer_task.execute + + # violations_channel.each do |v| + # Concurrent::Promises.future(v) do |v| + # puts "In future: Collecting violation details for #{v['infected_components']}: #{v['watch_name']} : #{v['issue_id']}" + # open(@pos_file, 'a') do |f| + # created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + # f.puts [created_date, v['watch_name'], v['issue_id']].join(',') + # end + + # pull_violation_details(v['violation_details_url']) + # end + # end sleep 100 # Need to add persistItem logic based on created_date @@ -252,26 +254,6 @@ def get_xray_violations_detail(xray_violation_detail_url) end - # queries the xray API for violations based upon the input json - def get_xray_violations(xray_json, jpd_url) - puts "jpd_url for #{jpd_url}" - response = RestClient::Request.new( - :method => :post, - :url => jpd_url + "/xray/api/v1/violations", - :payload => xray_json.to_json, - :user => @username, - :password => @apikey, - :headers => { :accept => :json, :content_type => :json} - ).execute do |response, request, result| - case response.code - when 200 - return response.to_str - else - puts "error: #{response.to_json}" - raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations. #{response.to_json}" - end - end - end # normalizes Xray data according to common information models for all log-vendors def data_normalization(detailResp) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/violations.json b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/violations.json new file mode 100644 index 0000000..0585695 --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/violations.json @@ -0,0 +1,380 @@ +{ + "total_violations":54, + "violations":[ + { + "description":"Apache Software Foundation (ASF) Struts before 1.2.9 allows remote attackers to bypass validation via a request with a 'org.apache.struts.taglib.html.Constants.CANCEL' parameter, which causes the action to be canceled but would not be detected from applications that do not use the isCancelled check.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.1" + ], + "created":"2021-06-16T21:22:18Z", + "watch_name":"maven-watch1", + "issue_id":"XRAY-55418", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=ff9cf61f42a095cd97ea0ec0&issue_id=XRAY-55418&comp_id=gav:%2F%2Fstruts:struts:1.1", + "impacted_artifacts":[ + "default/maven-repo-1/struts/struts/1.1/struts-1.1.jar" + ] + }, + { + "description":"The Apache Software License, Version 1.1", + "severity":"High", + "type":"License", + "infected_components":[ + "gav://struts:struts:1.1" + ], + "created":"2021-06-16T21:22:18Z", + "watch_name":"license-watch", + "issue_id":"Apache-1.1", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=b241c1986818d68993093e35&issue_id=Apache-1.1&comp_id=gav:%2F%2Fstruts:struts:1.1", + "impacted_artifacts":[ + "default/maven-repo-1/struts/struts/1.1/struts-1.1.jar" + ] + }, + { + "description":"The MultiPageValidator implementation in Apache Struts 1 1.1 through 1.3.10 allows remote attackers to bypass intended access restrictions via a modified page parameter.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.1" + ], + "created":"2021-06-16T21:22:18Z", + "watch_name":"maven-watch1", + "issue_id":"XRAY-55648", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=ff9cf61f42a095cd97ea0ec0&issue_id=XRAY-55648&comp_id=gav:%2F%2Fstruts:struts:1.1", + "impacted_artifacts":[ + "default/maven-repo-1/struts/struts/1.1/struts-1.1.jar" + ] + }, + { + "description":"Cross-site scripting (XSS) vulnerability in Apache Struts before 1.2.9-162.31.1 on SUSE Linux Enterprise (SLE) 11, before 1.2.9-108.2 on SUSE openSUSE 10.3, before 1.2.9-198.2 on SUSE openSUSE 11.0, and before 1.2.9-162.163.2 on SUSE openSUSE 11.1 allows remote attackers to inject arbitrary web script or HTML via unspecified vectors related to \"insufficient quoting of parameters.\"", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.1" + ], + "created":"2021-06-16T21:22:18Z", + "watch_name":"maven-watch1", + "issue_id":"XRAY-55444", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=ff9cf61f42a095cd97ea0ec0&issue_id=XRAY-55444&comp_id=gav:%2F%2Fstruts:struts:1.1", + "impacted_artifacts":[ + "default/maven-repo-1/struts/struts/1.1/struts-1.1.jar" + ] + }, + { + "description":"Cross-site scripting (XSS) vulnerability in (1) LookupDispatchAction and possibly (2) DispatchAction and (3) ActionDispatcher in Apache Software Foundation (ASF) Struts before 1.2.9 allows remote attackers to inject arbitrary web script or HTML via the parameter name, which is not filtered in the resulting error message.", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.1" + ], + "created":"2021-06-16T21:22:18Z", + "watch_name":"maven-watch1", + "issue_id":"XRAY-55420", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=ff9cf61f42a095cd97ea0ec0&issue_id=XRAY-55420&comp_id=gav:%2F%2Fstruts:struts:1.1", + "impacted_artifacts":[ + "default/maven-repo-1/struts/struts/1.1/struts-1.1.jar" + ] + }, + { + "description":"ActionForm in Apache Software Foundation (ASF) Struts before 1.2.9 with BeanUtils 1.7 allows remote attackers to cause a denial of service via a multipart/form-data encoded form with a parameter name that references the public getMultipartRequestHandler method, which provides further access to elements in the CommonsMultipartRequestHandler implementation and BeanUtils.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.1" + ], + "created":"2021-06-16T21:22:18Z", + "watch_name":"maven-watch1", + "issue_id":"XRAY-55419", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=ff9cf61f42a095cd97ea0ec0&issue_id=XRAY-55419&comp_id=gav:%2F%2Fstruts:struts:1.1", + "impacted_artifacts":[ + "default/maven-repo-1/struts/struts/1.1/struts-1.1.jar" + ] + }, + { + "description":"The MultiPageValidator implementation in Apache Struts 1 1.1 through 1.3.10 allows remote attackers to bypass intended access restrictions via a modified page parameter.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.2.4" + ], + "created":"2021-06-16T21:22:27Z", + "watch_name":"maven-watch-2", + "issue_id":"XRAY-55648", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2a3ee020bf1a86b84d122c0c&issue_id=XRAY-55648&comp_id=gav:%2F%2Fstruts:struts:1.2.4", + "impacted_artifacts":[ + "default/maven-repo-2/struts/struts/1.2.4/struts-1.2.4.jar" + ] + }, + { + "description":"ActionForm in Apache Software Foundation (ASF) Struts before 1.2.9 with BeanUtils 1.7 allows remote attackers to cause a denial of service via a multipart/form-data encoded form with a parameter name that references the public getMultipartRequestHandler method, which provides further access to elements in the CommonsMultipartRequestHandler implementation and BeanUtils.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.2.4" + ], + "created":"2021-06-16T21:22:27Z", + "watch_name":"maven-watch-2", + "issue_id":"XRAY-55419", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2a3ee020bf1a86b84d122c0c&issue_id=XRAY-55419&comp_id=gav:%2F%2Fstruts:struts:1.2.4", + "impacted_artifacts":[ + "default/maven-repo-2/struts/struts/1.2.4/struts-1.2.4.jar" + ] + }, + { + "description":"Cross-site scripting (XSS) vulnerability in (1) LookupDispatchAction and possibly (2) DispatchAction and (3) ActionDispatcher in Apache Software Foundation (ASF) Struts before 1.2.9 allows remote attackers to inject arbitrary web script or HTML via the parameter name, which is not filtered in the resulting error message.", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.2.4" + ], + "created":"2021-06-16T21:22:27Z", + "watch_name":"maven-watch-2", + "issue_id":"XRAY-55420", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2a3ee020bf1a86b84d122c0c&issue_id=XRAY-55420&comp_id=gav:%2F%2Fstruts:struts:1.2.4", + "impacted_artifacts":[ + "default/maven-repo-2/struts/struts/1.2.4/struts-1.2.4.jar" + ] + }, + { + "description":"The Apache Software License, Version 2.0", + "severity":"High", + "type":"License", + "infected_components":[ + "gav://struts:struts:1.2.4" + ], + "created":"2021-06-16T21:22:27Z", + "watch_name":"license-watch", + "issue_id":"Apache-2.0", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=b241c1986818d68993093e35&issue_id=Apache-2.0&comp_id=gav:%2F%2Fstruts:struts:1.2.4", + "impacted_artifacts":[ + "default/maven-repo-2/struts/struts/1.2.4/struts-1.2.4.jar" + ] + }, + { + "description":"Cross-site scripting (XSS) vulnerability in Apache Struts before 1.2.9-162.31.1 on SUSE Linux Enterprise (SLE) 11, before 1.2.9-108.2 on SUSE openSUSE 10.3, before 1.2.9-198.2 on SUSE openSUSE 11.0, and before 1.2.9-162.163.2 on SUSE openSUSE 11.1 allows remote attackers to inject arbitrary web script or HTML via unspecified vectors related to \"insufficient quoting of parameters.\"", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.2.4" + ], + "created":"2021-06-16T21:22:27Z", + "watch_name":"maven-watch-2", + "issue_id":"XRAY-55444", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2a3ee020bf1a86b84d122c0c&issue_id=XRAY-55444&comp_id=gav:%2F%2Fstruts:struts:1.2.4", + "impacted_artifacts":[ + "default/maven-repo-2/struts/struts/1.2.4/struts-1.2.4.jar" + ] + }, + { + "description":"Apache Software Foundation (ASF) Struts before 1.2.9 allows remote attackers to bypass validation via a request with a 'org.apache.struts.taglib.html.Constants.CANCEL' parameter, which causes the action to be canceled but would not be detected from applications that do not use the isCancelled check.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.2.4" + ], + "created":"2021-06-16T21:22:27Z", + "watch_name":"maven-watch-2", + "issue_id":"XRAY-55418", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2a3ee020bf1a86b84d122c0c&issue_id=XRAY-55418&comp_id=gav:%2F%2Fstruts:struts:1.2.4", + "impacted_artifacts":[ + "default/maven-repo-2/struts/struts/1.2.4/struts-1.2.4.jar" + ] + }, + { + "description":"Unicode Terms of Use", + "severity":"High", + "type":"License", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:37Z", + "watch_name":"license-watch", + "issue_id":"Unicode-TOU", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=b241c1986818d68993093e35&issue_id=Unicode-TOU&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"BSD 3-Clause \"New\" or \"Revised\" License", + "severity":"High", + "type":"License", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:37Z", + "watch_name":"license-watch", + "issue_id":"BSD-3-Clause", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=b241c1986818d68993093e35&issue_id=BSD-3-Clause&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"The Apache Software License, Version 2.0", + "severity":"High", + "type":"License", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:37Z", + "watch_name":"license-watch", + "issue_id":"Apache-2.0", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=b241c1986818d68993093e35&issue_id=Apache-2.0&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"The OGNL extensive expression evaluation capability in XWork in Struts 2.0.0 through 2.1.8.1, as used in Atlassian Fisheye, Crucible, and possibly other products, uses a permissive whitelist, which allows remote attackers to modify server-side context objects and bypass the \"#\" protection mechanism in ParameterInterceptors via the (1) #context, (2) #_memberAccess, (3) #root, (4) #this, (5) #_typeResolver, (6) #_classResolver, (7) #_traceEvaluations, (8) #_lastEvaluation, (9) #_keepLastEvaluation, and possibly other OGNL context variables, a different vulnerability than CVE-2008-6504.", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:37Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-55471", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-55471&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Academic Free License v2.1", + "severity":"High", + "type":"License", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:37Z", + "watch_name":"license-watch", + "issue_id":"AFL-2.1", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=b241c1986818d68993093e35&issue_id=AFL-2.1&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"ParametersInterceptor in OpenSymphony XWork 2.0.x before 2.0.6 and 2.1.x before 2.1.2, as used in Apache Struts and other products, does not properly restrict # (pound sign) references to context objects, which allows remote attackers to execute Object-Graph Navigation Language (OGNL) statements and modify server-side context objects, as demonstrated by use of a \\u0023 representation for the # character.", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-55446", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-55446&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Multiple cross-site scripting (XSS) vulnerabilities in Apache Struts 2.0.x before 2.0.11.1 and 2.1.x before 2.1.1 allow remote attackers to inject arbitrary web script or HTML via vectors associated with improper handling of (1) \" (double quote) characters in the href attribute of an s:a tag and (2) parameters in the action attribute of an s:url tag.", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-55448", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-55448&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Forced OGNL evaluation, when evaluated on raw user input in tag attributes, may lead to remote code execution. Affected software : Apache Struts 2.0.0 - Struts 2.5.25.", + "severity":"Critical", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-129844", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-129844&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"The ExceptionDelegator component in Apache Struts before 2.2.3.1 interprets parameter values as OGNL expressions during certain exception handling for mismatched data types of properties, which allows remote attackers to execute arbitrary Java code via a crafted parameter.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-55522", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-55522&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Apache Struts 2 before 2.3.14.3 allows remote attackers to execute arbitrary OGNL code via a request with a crafted value that contains both \"${}\" and \"%{}\" sequences, which causes the OGNL code to be evaluated twice.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-55575", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-55575&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Apache Struts JSP Page Handling Unspecified XSS", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-81164", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-81164&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Apache Struts Tag Handling XSS", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-81187", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-81187&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Apache Struts EL / OGNL Interpretation Unspecified Remote Code Execution", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-87424", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-87424&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + } + ] +} \ No newline at end of file diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb new file mode 100644 index 0000000..8102ca0 --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -0,0 +1,68 @@ +require 'concurrent' +require 'concurrent-edge' +require 'json' + +class Xray + def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file) + @jpd_url = jpd_url + @username = username + @api_key = api_key + @wait_interval = wait_interval + @batch_size = batch_size + @pos_file = pos_file + end + + def violations(date_since) + violations_channel = Concurrent::Channel.new(capacity: 100) + request_json = Concurrent::Channel.new(capacity: 1) + page_number = 1 + # timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do + xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } + resp = JSON.parse(get_xray_violations(xray_json)) + puts "Violations count is #{resp['total_violations']}" + resp['violations'].each do |v| + violations_channel << v + end + page_number += 1 + # end + # timer_task.execute + violations_channel + end + + def violation_details(violations_channel) + puts "violations details" + violations_channel.each do |v| + puts v + Concurrent::Promises.future(v) do |v| + puts "do nothing" + # open(@pos_file, 'a') do |f| + # created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + # f.puts [created_date, v['watch_name'], v['issue_id']].join(',') + # end + + # pull_violation_details(v['violation_details_url']) + end + end + end + + private + def get_xray_violations(xray_json) + puts "jpd_url for #{@jpd_url}" + response = RestClient::Request.new( + :method => :post, + :url => @jpd_url + "/xray/api/v1/violations", + :payload => xray_json.to_json, + :user => @username, + :password => @api_key, + :headers => { :accept => :json, :content_type => :json } + ).execute do |response, request, result| + case response.code + when 200 + return response.to_str + else + puts "error: #{response.to_json}" + raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations. #{response.to_json}" + end + end + end +end \ No newline at end of file diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/spec_helper.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/spec_helper.rb new file mode 100644 index 0000000..cb320f2 --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/spec_helper.rb @@ -0,0 +1,111 @@ +# This file was generated by the `rspec --init` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration + +[ + File.join(File.dirname(__FILE__), '..'), + File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'), + File.join(File.dirname(__FILE__), '..', 'spec'), +].each do |dir| + $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir) +end + +require 'xray' + +RSpec.configure do |config| + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + +# The settings below are suggested to provide a good initial experience +# with RSpec, but feel free to customize to your heart's content. +=begin + # This allows you to limit a spec run to individual examples or groups + # you care about by tagging them with `:focus` metadata. When nothing + # is tagged with `:focus`, all examples get run. RSpec also provides + # aliases for `it`, `describe`, and `context` that include `:focus` + # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + config.filter_run_when_matching :focus + + # Allows RSpec to persist some state between runs in order to support + # the `--only-failures` and `--next-failure` CLI options. We recommend + # you configure your source control system to ignore this file. + config.example_status_persistence_file_path = "spec/examples.txt" + + # Limits the available syntax to the non-monkey patched syntax that is + # recommended. For more details, see: + # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ + # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ + # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode + config.disable_monkey_patching! + + # This setting enables warnings. It's recommended, but in some cases may + # be too noisy due to issues in dependencies. + config.warnings = true + + # Many RSpec users commonly either run the entire suite or an individual + # file, and it's useful to allow more verbose output when running an + # individual spec file. + if config.files_to_run.one? + # Use the documentation formatter for detailed output, + # unless a formatter has already been configured + # (e.g. via a command-line flag). + config.default_formatter = "doc" + end + + # Print the 10 slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. + config.profile_examples = 10 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed +=end +end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb new file mode 100644 index 0000000..ddd7cba --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -0,0 +1,47 @@ +[ + File.join(File.dirname(__FILE__), '..'), + File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'), + File.join(File.dirname(__FILE__), '..', 'spec'), +].each do |dir| + $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir) +end + +require 'xray' +require 'date' +require 'rspec' + + +RSpec.describe Xray do + describe "#violation_details" do + it "creates a future for every item in the channel" do + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) + violations_channel = Concurrent::Channel.new(capacity: 5) + + (1..5).each do |i| + puts i + violations_channel << i + end + + promises = class_double("Concurrent::Promises") + expect(promises).to receive(:future).exactly(5).times + + xray.violation_details(violations_channel) + end + end + + describe "#violations" do + it "gets violations from date_since" do + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) + json = class_double(JSON) + expect(json).to receive(:parse).and_return({'violations': [1, 2, 3, 4, 5]}) + + violations_channel = xray.violations(Date.today) + result = [] + violations_channel.each do |v| + result << v + end + expect(result).to eq ([1, 2, 3, 4, 5]) + end + end + +end \ No newline at end of file diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/helper.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/helper.rb index 1562063..1959444 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/helper.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/helper.rb @@ -4,5 +4,6 @@ require "fluent/test/driver/input" require "fluent/test/helpers" + Test::Unit::TestCase.include(Fluent::Test::Helpers) Test::Unit::TestCase.extend(Fluent::Test::Helpers) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb index c77f319..2f84507 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb @@ -11,12 +11,22 @@ class JfrogSiemInputTest < Test::Unit::TestCase end # Default configuration for tests + # CONFIG = %[ + # tag "partnership.test_tag" + # jpd_url "https://partnership.jfrog.io/" + # username "sudhindrar" + # apikey "AKCp8ihpNg2JE5PV3nRXZQsmMGmzX9VTX6wN51hQBFRC1CXQWzGrKQvFL1tsw7aochjoQXAZq" + # pos_file "test_pos.txt" + # wait_interval 30 + # ] + CONFIG = %[ - tag "test_tag" - jpd_url JPD_URL - username USER - apikey API_KEY + tag "sudhindra-xray-rt.test_tag" + jpd_url "https://sudhindra-xray-rt.jfrog.tech/" + username "admin" + apikey "AKCp8jQd1zP4oKv43SNgewrNwikd1iAQznfhSfx3T249eVMkGnJnSjCpNsuv8vtHWChKLfJ1w" pos_file "test_pos.txt" + wait_interval 10 ] private From d499fa210294565c574ed54890e91ba73f5dfe6d Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Fri, 25 Jun 2021 15:32:18 -0700 Subject: [PATCH 10/44] Fixes the logic to use known length arrays instead of channels - fixes rspec setup so that `rspec .` works - fixes the setup so that integration test works with `rake test` - 4 green specs and lots of refactored code --- .../input/fluent-plugin-jfrog-siem/Rakefile | 2 +- .../lib/fluent/plugin/in_jfrog_siem.rb | 17 ++++-- .../lib/fluent/plugin/xray.rb | 54 ++++++++++--------- .../spec/xray_spec.rb | 53 ++++++++++++------ 4 files changed, 79 insertions(+), 47 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/Rakefile b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Rakefile index 9caf936..a388768 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/Rakefile +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Rakefile @@ -4,7 +4,7 @@ Bundler::GemHelper.install_tasks require "rake/testtask" Rake::TestTask.new(:test) do |t| - t.libs.push("lib", "test") + t.libs.push("lib", "test", 'lib/fluent/plugin') t.test_files = FileList["test/**/test_*.rb"] t.verbose = true t.warning = false diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 5b336ab..076b6fe 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. require "fluent/plugin/input" -require "rest-client" require "date" require "uri" require 'xray' @@ -32,7 +31,7 @@ class JfrogSiemInput < Fluent::Plugin::Input config_param :username, :string, default: "" config_param :apikey, :string, default: "" config_param :pos_file, :string, default: "" - config_param :batch_size, :integer, default: 25 + config_param :batch_size, :integer, default: 5 config_param :thread_count, :integer, default: 5 config_param :wait_interval, :integer, default: 60 @@ -101,12 +100,20 @@ def run rescue last_created_date = DateTime.parse("1970-01-01T00:00:00Z").strftime("%Y-%m-%dT%H:%M:%SZ") end - date_since = last_created_date + for_date = last_created_date xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) - violations_channel = xray.violations(date_since) - xray.violation_details(violations_channel) + violations_count = xray.violations_count(for_date) + puts violations_count + puts xray.page_count(violations_count) + page_count = xray.page_count(violations_count) + (1..xray.page_count(violations_count)).each do |page_number| + violations = xray.violations_by_page(for_date, page_number) + puts "getting details for #{page_number}" + xray.violation_details(violations) + end + # violations_channel = Concurrent::Channel.new(capacity: 100) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 8102ca0..e365c91 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -1,6 +1,8 @@ require 'concurrent' require 'concurrent-edge' require 'json' +require "rest-client" + class Xray def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file) @@ -12,35 +14,37 @@ def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file) @pos_file = pos_file end - def violations(date_since) - violations_channel = Concurrent::Channel.new(capacity: 100) - request_json = Concurrent::Channel.new(capacity: 1) - page_number = 1 - # timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do - xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } - resp = JSON.parse(get_xray_violations(xray_json)) - puts "Violations count is #{resp['total_violations']}" - resp['violations'].each do |v| - violations_channel << v - end - page_number += 1 - # end - # timer_task.execute - violations_channel + def violations_count(for_date) + xray_json = {"filters": { "created_from": for_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": 1 } } + JSON.parse(get_xray_violations(xray_json))['total_violations'] + end + + def page_count(total_violations) + pages = total_violations / @batch_size + another_page = total_violations % @batch_size + return pages + 1 if another_page > 0 + return pages end - def violation_details(violations_channel) - puts "violations details" - violations_channel.each do |v| - puts v + def violations_by_page(for_date, page_number) + violations = Concurrent::Array.new(@batch_size) + xray_json = {"filters": { "created_from": for_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } + resp = JSON.parse(get_xray_violations(xray_json)) + resp['violations'].each do |v| + violations << v + end + violations + end + + def violation_details(violations) + violations.each do |v| Concurrent::Promises.future(v) do |v| - puts "do nothing" - # open(@pos_file, 'a') do |f| - # created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") - # f.puts [created_date, v['watch_name'], v['issue_id']].join(',') - # end + open(@pos_file, 'a') do |f| + created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + f.puts [created_date, v['watch_name'], v['issue_id']].join(',') + end - # pull_violation_details(v['violation_details_url']) + pull_violation_details(v['violation_details_url']) end end end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index ddd7cba..aada386 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -15,33 +15,54 @@ describe "#violation_details" do it "creates a future for every item in the channel" do xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) - violations_channel = Concurrent::Channel.new(capacity: 5) + violations = Concurrent::Array.new (1..5).each do |i| - puts i - violations_channel << i + violations << i end - promises = class_double("Concurrent::Promises") + promises = class_double("Concurrent::Promises").as_stubbed_const(:transfer_nested_constants => true) expect(promises).to receive(:future).exactly(5).times - xray.violation_details(violations_channel) + xray.violation_details(violations) end end - describe "#violations" do - it "gets violations from date_since" do - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) - json = class_double(JSON) - expect(json).to receive(:parse).and_return({'violations': [1, 2, 3, 4, 5]}) + # Need to fix expectation on the instance_double to fix this spec + #describe "#violations_by_page" do + # it "gets violations for for_date" do + # xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, 5, @pos_file) + # + # rest_client = instance_double(RestClient::Request) + # expect(rest_client).to receive(:execute).and_return({"violations": [1, 2, 3, 4, 5]}) - violations_channel = xray.violations(Date.today) - result = [] - violations_channel.each do |v| - result << v - end - expect(result).to eq ([1, 2, 3, 4, 5]) + # json = class_double(JSON) + # expect(json).to receive(:parse).and_return({'violations': [1, 2, 3, 4, 5]}) + + # violations = xray.violations_by_page(Date.today, 1) + # expect(violations).to eq ([1, 2, 3, 4, 5]) + # end + # end + + describe "#page_count" do + it "calculates page_count based on batch_size to account for last page smaller than batch_size" do + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, 5, @pos_file) + + expect(xray.page_count(24)).to be(5) + end + + it "calculates page_count based on batch_size to account for last page same as batch_size" do + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, 5, @pos_file) + + expect(xray.page_count(20)).to be(4) + end + + it "returns elegantly when violations_count is 0" do + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, 5, @pos_file) + + expect(xray.page_count(0)).to be(0) end + end end \ No newline at end of file From b72c2ba5e2e7d5479d8540c56c5562d3de9874a4 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Tue, 29 Jun 2021 13:59:04 -0700 Subject: [PATCH 11/44] Moves methods into xray class - since they are used only in xray class - also makes them private to encapsulate better. --- .../lib/fluent/plugin/in_jfrog_siem.rb | 198 ------------------ .../lib/fluent/plugin/xray.rb | 91 ++++++++ 2 files changed, 91 insertions(+), 198 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 076b6fe..35d3afe 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -113,112 +113,8 @@ def run puts "getting details for #{page_number}" xray.violation_details(violations) end - - - - # violations_channel = Concurrent::Channel.new(capacity: 100) - # timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do - # xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } - # resp = JSON.parse(get_xray_violations(xray_json, @jpd_url)) - # puts "Violations count is #{resp['total_violations']}" - # resp['violations'].each do |v| - # violations_channel << v - # end - # page_number += 1 - # end - # timer_task.execute - - # violations_channel.each do |v| - # Concurrent::Promises.future(v) do |v| - # puts "In future: Collecting violation details for #{v['infected_components']}: #{v['watch_name']} : #{v['issue_id']}" - # open(@pos_file, 'a') do |f| - # created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") - # f.puts [created_date, v['watch_name'], v['issue_id']].join(',') - # end - - # pull_violation_details(v['violation_details_url']) - # end - # end - sleep 100 - # Need to add persistItem logic based on created_date - - # while true - # # Grab the batch of records - # resp=get_xray_violations(xray_json, @jpd_url) - # number_of_violations = JSON.parse(resp)['total_violations'] - # if left_violations <= 0 - # left_violations = number_of_violations - # end - - # xray_violation_urls_list = [] - # for index in 0..JSON.parse(resp)['violations'].length-1 do - # # Get the violation - # item = JSON.parse(resp)['violations'][index] - - # # Get the created date and check if we should skip (already processed) or process this record. - # created_date_string = item['created'] - # created_date = DateTime.parse(created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ") - - # # Determine if we need to persist this record or not - # persistItem = true - # if waiting_for_violations - # if created_date <= last_created_date - # # "not persisting it - waiting for violations" - # persistItem = false - # end - # else - # if created_date < last_created_date - # # "persisting everything" - # persistItem = true - # end - # end - - # # Publish the record to fluentd - # if persistItem - - # now = Fluent::Engine.now - # router.emit(@tag, now, item) - - # # write to the pos_file created_date_string - # open(@pos_file, 'a') do |f| - # f << "#{created_date_string}\n" - # end - - # # Mark this as the last record successfully processed - # last_created_date_string = created_date_string - # last_created_date = created_date - - # # Grab violation detail url and add to url list to process w/ thread pool - # xray_violation_details_url=item['violation_details_url'] - # xray_violation_urls_list.append(xray_violation_details_url) - # end - # end - - # xray_violation_urls_list.map do |xv_url| - # Concurrent::Promises.future(xv_url)) { |xv| pull_violation_details xv } - # end - - # begin - # xray_violation_urls_list.value!.map(&:value!) - # rescue => e - # puts "Failed to pull violation details due to #{e}" - # end - - # # reduce left violations by jump size (not all batches have full item count??) - # left_violations = left_violations - @batch_size - # if left_violations <= 0 - # waiting_for_violations = true - # sleep(@wait_interval) - # else - # # Grab the next record to process for the violation details url - # waiting_for_violations = false - # offset_count = offset_count + 1 - # xray_json={"filters": { "created_from": last_created_date_string }, "pagination": {"order_by": "created","limit": @batch_size , "offset": offset_count } } - # end - # end end - # pull the last item create date from the pos_file return created_date_string def get_last_item_create_date() if(!(File.exist?(@pos_file))) @@ -242,100 +138,6 @@ def call_home(jpd_url) end end - # queries the xray API for violations based upon the input json - def get_xray_violations_detail(xray_violation_detail_url) - response = RestClient::Request.new( - :method => :get, - :url => xray_violation_detail_url, - :user => @username, - :password => @apikey - ).execute do |response, request, result| - case response.code - when 200 - return response.to_str - else - puts "error: #{response.to_json}" - raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." - end - end - end - - - - # normalizes Xray data according to common information models for all log-vendors - def data_normalization(detailResp) - detailResp_json = JSON.parse(detailResp) - cve = [] - cvss_v2_list = [] - cvss_v3_list = [] - policy_list = [] - rule_list = [] - impacted_artifact_url_list = [] - if detailResp_json.key?('properties') - properties = detailResp_json['properties'] - for index in 0..properties.length-1 do - if properties[index].key?('cve') - cve.push(properties[index]['cve']) - end - if properties[index].key?('cvss_v2') - cvss_v2_list.push(properties[index]['cvss_v2']) - end - if properties[index].key?('cvss_v3') - cvss_v3_list.push(properties[index]['cvss_v3']) - end - end - - detailResp_json["cve"] = cve.sort.reverse[0] - cvss_v2 = cvss_v2_list.sort.reverse[0] - cvss_v3 = cvss_v3_list.sort.reverse[0] - if !cvss_v3.nil? - cvss = cvss_v3 - elsif !cvss_v2.nil? - cvss = cvss_v2 - end - cvss_score = cvss[0..2] - cvss_version = cvss.split(':')[1][0..2] - detailResp_json["cvss_score"] = cvss_score - detailResp_json["cvss_version"] = cvss_version - end - - if detailResp_json.key?('matched_policies') - matched_policies = detailResp_json['matched_policies'] - for index in 0..matched_policies.length-1 do - if matched_policies[index].key?('policy') - policy_list.push(matched_policies[index]['policy']) - end - if matched_policies[index].key?('rule') - rule_list.push(matched_policies[index]['rule']) - end - end - detailResp_json['policies'] = policy_list - detailResp_json['rules'] = rule_list - end - - impacted_artifacts = detailResp_json['impacted_artifacts'] - for impacted_artifact in impacted_artifacts do - matchdata = impacted_artifact.match /default\/(?[^\/]*)\/(?.*)/ - impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " " - impacted_artifact_url_list.append(impacted_artifact_url) - end - detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list - return detailResp_json - end - - def pull_violation_details(xray_violation_detail_url) - begin - puts "Pulling violation details for #{xray_violation_detail_url}" - detailResp=get_xray_violations_detail(xray_violation_detail_url) - time = Fluent::Engine.now - detailResp_json = data_normalization(detailResp) - router.emit(@tag, time, detailResp_json) - rescue => e - puts "error: #{e}" - raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" - end - end - end end end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index e365c91..fe4f904 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -69,4 +69,95 @@ def get_xray_violations(xray_json) end end end + # queries the xray API for violations based upon the input json + def get_xray_violations_detail(xray_violation_detail_url) + response = RestClient::Request.new( + :method => :get, + :url => xray_violation_detail_url, + :user => @username, + :password => @apikey + ).execute do |response, request, result| + case response.code + when 200 + return response.to_str + else + puts "error: #{response.to_json}" + raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." + end + end + end + + # normalizes Xray data according to common information models for all log-vendors + def data_normalization(detailResp) + detailResp_json = JSON.parse(detailResp) + cve = [] + cvss_v2_list = [] + cvss_v3_list = [] + policy_list = [] + rule_list = [] + impacted_artifact_url_list = [] + if detailResp_json.key?('properties') + properties = detailResp_json['properties'] + for index in 0..properties.length-1 do + if properties[index].key?('cve') + cve.push(properties[index]['cve']) + end + if properties[index].key?('cvss_v2') + cvss_v2_list.push(properties[index]['cvss_v2']) + end + if properties[index].key?('cvss_v3') + cvss_v3_list.push(properties[index]['cvss_v3']) + end + end + + detailResp_json["cve"] = cve.sort.reverse[0] + cvss_v2 = cvss_v2_list.sort.reverse[0] + cvss_v3 = cvss_v3_list.sort.reverse[0] + if !cvss_v3.nil? + cvss = cvss_v3 + elsif !cvss_v2.nil? + cvss = cvss_v2 + end + cvss_score = cvss[0..2] + cvss_version = cvss.split(':')[1][0..2] + detailResp_json["cvss_score"] = cvss_score + detailResp_json["cvss_version"] = cvss_version + end + + if detailResp_json.key?('matched_policies') + matched_policies = detailResp_json['matched_policies'] + for index in 0..matched_policies.length-1 do + if matched_policies[index].key?('policy') + policy_list.push(matched_policies[index]['policy']) + end + if matched_policies[index].key?('rule') + rule_list.push(matched_policies[index]['rule']) + end + end + detailResp_json['policies'] = policy_list + detailResp_json['rules'] = rule_list + end + + impacted_artifacts = detailResp_json['impacted_artifacts'] + for impacted_artifact in impacted_artifacts do + matchdata = impacted_artifact.match /default\/(?[^\/]*)\/(?.*)/ + impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " " + impacted_artifact_url_list.append(impacted_artifact_url) + end + detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list + return detailResp_json + end + + def pull_violation_details(xray_violation_detail_url) + begin + puts "Pulling violation details for #{xray_violation_detail_url}" + detailResp=get_xray_violations_detail(xray_violation_detail_url) + time = Fluent::Engine.now + detailResp_json = data_normalization(detailResp) + router.emit(@tag, time, detailResp_json) + rescue => e + puts "error: #{e}" + raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" + end + end end \ No newline at end of file From f0b73e46cc94166bcd41c6739c7e2e41b3d0ed7a Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Tue, 6 Jul 2021 22:23:49 -0700 Subject: [PATCH 12/44] Makes a test green using the correct doubles - removes array size initializer since it creates array of nils - fixes json parsing to be consistent --- .../fluent-plugin-jfrog-siem/Gemfile.lock | 4 +-- .../lib/fluent/plugin/xray.rb | 8 ++--- .../spec/xray_spec.rb | 29 +++++++++---------- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock index efea5f2..b872189 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - fluent-plugin-jfrog-siem (0.1.8) + fluent-plugin-jfrog-siem (1.0.0) concurrent-ruby (~> 1.1.8) concurrent-ruby-edge fluentd (>= 0.14.10, < 2) @@ -16,7 +16,7 @@ GEM diff-lcs (1.4.4) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - fluentd (1.13.0) + fluentd (1.13.1) bundler cool.io (>= 1.4.5, < 2.0.0) http_parser.rb (>= 0.5.1, < 0.7.0) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index fe4f904..672830c 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -27,10 +27,10 @@ def page_count(total_violations) end def violations_by_page(for_date, page_number) - violations = Concurrent::Array.new(@batch_size) + violations = Concurrent::Array.new xray_json = {"filters": { "created_from": for_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } - resp = JSON.parse(get_xray_violations(xray_json)) - resp['violations'].each do |v| + resp = JSON.parse(get_xray_violations(xray_json), {symbolize_names: true}) + resp[:violations].each do |v| violations << v end violations @@ -62,7 +62,7 @@ def get_xray_violations(xray_json) ).execute do |response, request, result| case response.code when 200 - return response.to_str + return response else puts "error: #{response.to_json}" raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations. #{response.to_json}" diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index aada386..013b03b 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -28,21 +28,20 @@ end end - # Need to fix expectation on the instance_double to fix this spec - #describe "#violations_by_page" do - # it "gets violations for for_date" do - # xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, 5, @pos_file) - # - # rest_client = instance_double(RestClient::Request) - # expect(rest_client).to receive(:execute).and_return({"violations": [1, 2, 3, 4, 5]}) - - # json = class_double(JSON) - # expect(json).to receive(:parse).and_return({'violations': [1, 2, 3, 4, 5]}) - - # violations = xray.violations_by_page(Date.today, 1) - # expect(violations).to eq ([1, 2, 3, 4, 5]) - # end - # end + describe "#violations_by_page" do + it "gets violations for for_date" do + xray = Xray.new("@jpd_url", @username, @apikey, @wait_interval, 5, @pos_file) + + rest_client = double("A Rest Client") + expect(RestClient::Request).to receive(:new).and_return rest_client + expect(rest_client).to receive(:execute).and_return('{"violations": [1, 2, 3, 4, 5]}') + + expect(JSON).to receive(:parse).and_return({'violations': [1, 2, 3, 4, 5]}) + + violations = xray.violations_by_page(Date.today, 1) + expect(violations).to eq ([1, 2, 3, 4, 5]) + end + end describe "#page_count" do it "calculates page_count based on batch_size to account for last page smaller than batch_size" do From c8f0e97de013978a8000effb819399e649533791 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Thu, 8 Jul 2021 11:27:08 -0700 Subject: [PATCH 13/44] Pulls violations from a starting date - fixes issues with retrieving properties for violations --- .../lib/fluent/plugin/in_jfrog_siem.rb | 22 +++++++++++------- .../lib/fluent/plugin/xray.rb | 10 ++++---- .../spec/xray_spec.rb | 23 ++++++++++++++++++- .../test/plugin/test_in_jfrog_siem.rb | 2 ++ 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 35d3afe..b9f97e9 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -34,6 +34,7 @@ class JfrogSiemInput < Fluent::Plugin::Input config_param :batch_size, :integer, default: 5 config_param :thread_count, :integer, default: 5 config_param :wait_interval, :integer, default: 60 + config_param :start_date, :string, default: Date.today.to_s # `configure` is called before `start`. @@ -104,15 +105,20 @@ def run xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) - violations_count = xray.violations_count(for_date) - puts violations_count - puts xray.page_count(violations_count) - page_count = xray.page_count(violations_count) - (1..xray.page_count(violations_count)).each do |page_number| - violations = xray.violations_by_page(for_date, page_number) - puts "getting details for #{page_number}" - xray.violation_details(violations) + DateTime.parse(start_date).upto(Date.today) do |for_date| + # Concurrent::Promises.future(for_date) do |for_date| + violations_count = xray.violations_count(for_date) + puts violations_count + puts xray.page_count(violations_count) + page_count = xray.page_count(violations_count) + (1..xray.page_count(violations_count)).each do |page_number| + violations = xray.violations_by_page(for_date, page_number) + puts "getting details for #{page_number}" + xray.violation_details(violations) + end + # end end + end # pull the last item create date from the pos_file return created_date_string diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 672830c..96208d3 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -39,19 +39,19 @@ def violations_by_page(for_date, page_number) def violation_details(violations) violations.each do |v| Concurrent::Promises.future(v) do |v| - open(@pos_file, 'a') do |f| - created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") - f.puts [created_date, v['watch_name'], v['issue_id']].join(',') + puts "In future: ", v + File.open(@pos_file, 'a') do |f| + timestamp = DateTime.parse(v[:created]).strftime("%Y-%m-%dT%H:%M:%SZ") + f.puts [timestamp, v[:watch_name], v[:issue_id]].join(',') end - pull_violation_details(v['violation_details_url']) + pull_violation_details(v[:violation_details_url]) end end end private def get_xray_violations(xray_json) - puts "jpd_url for #{@jpd_url}" response = RestClient::Request.new( :method => :post, :url => @jpd_url + "/xray/api/v1/violations", diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index 013b03b..1059f19 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -13,7 +13,7 @@ RSpec.describe Xray do describe "#violation_details" do - it "creates a future for every item in the channel" do + it "creates a future for every violation" do xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) violations = Concurrent::Array.new @@ -26,6 +26,27 @@ xray.violation_details(violations) end + + xit "updates pos file for every violation" do + pos_file = double('pos_file') + + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, "pos_file.txt") + violations = Concurrent::Array.new + + (1..5).each do |i| + violations << i + end + + datetime = double (DateTime) + expect(datetime).to receive(:parse) + promises = class_double("Concurrent::Promises").as_stubbed_const(:transfer_nested_constants => true) + allow(promises).to receive(:future) { |&block| block.call } + + expect(File).to receive(:open).with("pos_file.txt", "a").and_yield(pos_file) + expect(pos_file).to receive(:puts).exactly(5).times + + xray.violation_details(violations) + end end describe "#violations_by_page" do diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb index 2f84507..323ce0e 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb @@ -27,6 +27,8 @@ class JfrogSiemInputTest < Test::Unit::TestCase apikey "AKCp8jQd1zP4oKv43SNgewrNwikd1iAQznfhSfx3T249eVMkGnJnSjCpNsuv8vtHWChKLfJ1w" pos_file "test_pos.txt" wait_interval 10 + start_date "2021-06-15" + batch_size 25 ] private From d1adbf4aa55dfc3a10256e82621eec7be7221eb7 Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Mon, 12 Jul 2021 14:05:18 -0700 Subject: [PATCH 14/44] Pushing live+historic code --- .../lib/fluent/plugin/in_jfrog_siem.rb | 33 ++- .../lib/fluent/plugin/xray.rb | 272 ++++++++++-------- 2 files changed, 168 insertions(+), 137 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 35d3afe..0d2a089 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. require "fluent/plugin/input" +require "rest-client" require "date" require "uri" -require 'xray' +require "fluent/plugin/xray.rb" module Fluent module Plugin @@ -31,10 +32,10 @@ class JfrogSiemInput < Fluent::Plugin::Input config_param :username, :string, default: "" config_param :apikey, :string, default: "" config_param :pos_file, :string, default: "" - config_param :batch_size, :integer, default: 5 + config_param :batch_size, :integer, default: 25 config_param :thread_count, :integer, default: 5 config_param :wait_interval, :integer, default: 60 - + config_param :from_date, :string, default: "" # `configure` is called before `start`. # 'conf' is a `Hash` that includes the configuration parameters. @@ -98,21 +99,19 @@ def run begin last_created_date = DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ") rescue - last_created_date = DateTime.parse("1970-01-01T00:00:00Z").strftime("%Y-%m-%dT%H:%M:%SZ") + last_created_date = DateTime.now.strftime("%Y-%m-%dT%H:%M:%SZ") end - for_date = last_created_date - - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) - violations_count = xray.violations_count(for_date) - puts violations_count - puts xray.page_count(violations_count) - page_count = xray.page_count(violations_count) - (1..xray.page_count(violations_count)).each do |page_number| - violations = xray.violations_by_page(for_date, page_number) - puts "getting details for #{page_number}" - xray.violation_details(violations) + if (@from_date != "") + last_created_date = DateTime.parse(@from_date).strftime("%Y-%m-%dT%H:%M:%SZ") end + date_since = last_created_date + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) + violations_channel = xray.violations(date_since) + xray.violation_details(violations_channel) + + sleep 100 + end # pull the last item create date from the pos_file return created_date_string @@ -134,10 +133,10 @@ def call_home(jpd_url) :password => @apikey, :headers => { :accept => :json, :content_type => :json} ).execute do |response, request, result| - puts "Posting call home information" + puts "Posting call home information" end end - end end end + diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index fe4f904..890e30d 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -1,8 +1,6 @@ require 'concurrent' require 'concurrent-edge' require 'json' -require "rest-client" - class Xray def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file) @@ -14,150 +12,184 @@ def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file) @pos_file = pos_file end - def violations_count(for_date) - xray_json = {"filters": { "created_from": for_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": 1 } } - JSON.parse(get_xray_violations(xray_json))['total_violations'] - end + def violations(date_since) + violations_channel = Concurrent::Channel.new(capacity: @batch_size) + page_number = 1 + timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do + xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } + resp = JSON.parse(get_xray_violations(xray_json)) + total_violation_count = resp['total_violations'] + page_violation_count = resp['violations'].length + puts "Total violations count is #{total_violation_count}" + if total_violation_count > 0 + puts "Number of Violations in page #{page_number} are #{page_violation_count}" + last_limit_lines = get_last_limit_lines_from_pos_file(page_violation_count) - def page_count(total_violations) - pages = total_violations / @batch_size - another_page = total_violations % @batch_size - return pages + 1 if another_page > 0 - return pages + if is_pos_file_empty + resp['violations'].each do |v| + violations_channel << v + end + else + resp['violations'].each do |v| + alreadyProcessed = check_if_violation_already_processed(v, last_limit_lines) + if !alreadyProcessed + puts "Not processed" + violations_channel << v + else + puts "Already processed" + end + end + end + if page_violation_count == @batch_size + page_number += 1 + end + end + end + timer_task.execute + violations_channel end - def violations_by_page(for_date, page_number) - violations = Concurrent::Array.new(@batch_size) - xray_json = {"filters": { "created_from": for_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } - resp = JSON.parse(get_xray_violations(xray_json)) - resp['violations'].each do |v| - violations << v + def is_pos_file_empty() + file_lines = File.foreach(@pos_file).count + if file_lines == 0 + return true end - violations + return false + end + + def check_if_violation_already_processed(v, last_limit_lines) + v_line = [DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ"), v['watch_name'], v['issue_id']].join(',') + return last_limit_lines.any? { |s| s.include?(v_line) } end - def violation_details(violations) - violations.each do |v| - Concurrent::Promises.future(v) do |v| + def get_last_limit_lines_from_pos_file(page_violation_count) + last_limit_lines = IO.readlines(@pos_file)[-page_violation_count..-1] + return last_limit_lines + end + + def violation_details(violations_channel) + # emit only violation details and not all + puts "violations details" + violations_channel.each do |v| + Concurrent::Promises.future(v) do |v7| + pull_violation_details(v['violation_details_url']) open(@pos_file, 'a') do |f| created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") f.puts [created_date, v['watch_name'], v['issue_id']].join(',') end - - pull_violation_details(v['violation_details_url']) end end end - private - def get_xray_violations(xray_json) - puts "jpd_url for #{@jpd_url}" - response = RestClient::Request.new( - :method => :post, - :url => @jpd_url + "/xray/api/v1/violations", - :payload => xray_json.to_json, - :user => @username, - :password => @api_key, - :headers => { :accept => :json, :content_type => :json } - ).execute do |response, request, result| - case response.code - when 200 - return response.to_str - else - puts "error: #{response.to_json}" - raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations. #{response.to_json}" - end - end + def pull_violation_details(xray_violation_detail_url) + begin + #puts "Pulling violation details for #{xray_violation_detail_url}" + detailResp=get_xray_violations_detail(xray_violation_detail_url) + time = Fluent::Engine.now + detailResp_json = data_normalization(detailResp) + #puts detailResp_json + #router.emit(@tag, time, detailResp_json) + rescue => e + puts "error1: #{e}" + raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" end - # queries the xray API for violations based upon the input json - def get_xray_violations_detail(xray_violation_detail_url) - response = RestClient::Request.new( - :method => :get, - :url => xray_violation_detail_url, - :user => @username, - :password => @apikey - ).execute do |response, request, result| - case response.code - when 200 - return response.to_str - else - puts "error: #{response.to_json}" - raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." - end + end + + def get_xray_violations_detail(xray_violation_detail_url) + response = RestClient::Request.new( + :method => :get, + :url => xray_violation_detail_url, + :user => @username, + :password => @api_key + ).execute do |response, request, result| + case response.code + when 200 + return response.to_str + else + puts "error2: #{response.to_json}" + raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." end end + end - # normalizes Xray data according to common information models for all log-vendors - def data_normalization(detailResp) - detailResp_json = JSON.parse(detailResp) - cve = [] - cvss_v2_list = [] - cvss_v3_list = [] - policy_list = [] - rule_list = [] - impacted_artifact_url_list = [] - if detailResp_json.key?('properties') - properties = detailResp_json['properties'] - for index in 0..properties.length-1 do - if properties[index].key?('cve') - cve.push(properties[index]['cve']) - end - if properties[index].key?('cvss_v2') - cvss_v2_list.push(properties[index]['cvss_v2']) - end - if properties[index].key?('cvss_v3') - cvss_v3_list.push(properties[index]['cvss_v3']) - end + def data_normalization(detailResp) + detailResp_json = JSON.parse(detailResp) + cve = [] + cvss_v2_list = [] + cvss_v3_list = [] + policy_list = [] + rule_list = [] + impacted_artifact_url_list = [] + if detailResp_json.key?('properties') + properties = detailResp_json['properties'] + for index in 0..properties.length-1 do + if properties[index].key?('cve') + cve.push(properties[index]['cve']) end - - detailResp_json["cve"] = cve.sort.reverse[0] - cvss_v2 = cvss_v2_list.sort.reverse[0] - cvss_v3 = cvss_v3_list.sort.reverse[0] - if !cvss_v3.nil? - cvss = cvss_v3 - elsif !cvss_v2.nil? - cvss = cvss_v2 + if properties[index].key?('cvss_v2') + cvss_v2_list.push(properties[index]['cvss_v2']) + end + if properties[index].key?('cvss_v3') + cvss_v3_list.push(properties[index]['cvss_v3']) end - cvss_score = cvss[0..2] - cvss_version = cvss.split(':')[1][0..2] - detailResp_json["cvss_score"] = cvss_score - detailResp_json["cvss_version"] = cvss_version end - if detailResp_json.key?('matched_policies') - matched_policies = detailResp_json['matched_policies'] - for index in 0..matched_policies.length-1 do - if matched_policies[index].key?('policy') - policy_list.push(matched_policies[index]['policy']) - end - if matched_policies[index].key?('rule') - rule_list.push(matched_policies[index]['rule']) - end - end - detailResp_json['policies'] = policy_list - detailResp_json['rules'] = rule_list + detailResp_json["cve"] = cve.sort.reverse[0] + cvss_v2 = cvss_v2_list.sort.reverse[0] + cvss_v3 = cvss_v3_list.sort.reverse[0] + if !cvss_v3.nil? + cvss = cvss_v3 + elsif !cvss_v2.nil? + cvss = cvss_v2 end + cvss_score = cvss[0..2] + cvss_version = cvss.split(':')[1][0..2] + detailResp_json["cvss_score"] = cvss_score + detailResp_json["cvss_version"] = cvss_version + end - impacted_artifacts = detailResp_json['impacted_artifacts'] - for impacted_artifact in impacted_artifacts do - matchdata = impacted_artifact.match /default\/(?[^\/]*)\/(?.*)/ - impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " " - impacted_artifact_url_list.append(impacted_artifact_url) + if detailResp_json.key?('matched_policies') + matched_policies = detailResp_json['matched_policies'] + for index in 0..matched_policies.length-1 do + if matched_policies[index].key?('policy') + policy_list.push(matched_policies[index]['policy']) + end + if matched_policies[index].key?('rule') + rule_list.push(matched_policies[index]['rule']) + end end - detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list - return detailResp_json + detailResp_json['policies'] = policy_list + detailResp_json['rules'] = rule_list + end + + impacted_artifacts = detailResp_json['impacted_artifacts'] + for impacted_artifact in impacted_artifacts do + matchdata = impacted_artifact.match /default\/(?[^\/]*)\/(?.*)/ + impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " " + impacted_artifact_url_list.append(impacted_artifact_url) end + detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list + return detailResp_json + end - def pull_violation_details(xray_violation_detail_url) - begin - puts "Pulling violation details for #{xray_violation_detail_url}" - detailResp=get_xray_violations_detail(xray_violation_detail_url) - time = Fluent::Engine.now - detailResp_json = data_normalization(detailResp) - router.emit(@tag, time, detailResp_json) - rescue => e - puts "error: #{e}" - raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" + private + def get_xray_violations(xray_json) + response = RestClient::Request.new( + :method => :post, + :url => @jpd_url + "/xray/api/v1/violations", + :payload => xray_json.to_json, + :user => @username, + :password => @api_key, + :headers => { :accept => :json, :content_type => :json } + ).execute do |response, request, result| + case response.code + when 200 + return response.to_str + else + puts "error3: #{response.to_json}" + raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations. #{response.to_json}" end end -end \ No newline at end of file + end +end + From 6fa046e5df8b3962dc0b06d0a1ffaa86155530f0 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Mon, 12 Jul 2021 22:02:18 -0700 Subject: [PATCH 15/44] Uses ruby function to test file for emptiness - removes the corresponding function --- .../fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 890e30d..331b489 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -25,7 +25,7 @@ def violations(date_since) puts "Number of Violations in page #{page_number} are #{page_violation_count}" last_limit_lines = get_last_limit_lines_from_pos_file(page_violation_count) - if is_pos_file_empty + if File.zero?(@pos_file) resp['violations'].each do |v| violations_channel << v end @@ -49,14 +49,6 @@ def violations(date_since) violations_channel end - def is_pos_file_empty() - file_lines = File.foreach(@pos_file).count - if file_lines == 0 - return true - end - return false - end - def check_if_violation_already_processed(v, last_limit_lines) v_line = [DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ"), v['watch_name'], v['issue_id']].join(',') return last_limit_lines.any? { |s| s.include?(v_line) } From ede9a6797816a9c3ccce357d783df5c68120e52c Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Tue, 13 Jul 2021 00:34:29 -0700 Subject: [PATCH 16/44] Adding pos file per date --- .../lib/fluent/plugin/in_jfrog_siem.rb | 2 +- .../lib/fluent/plugin/xray.rb | 56 +++++++++++-------- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 0d2a089..ddd8262 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -31,7 +31,7 @@ class JfrogSiemInput < Fluent::Plugin::Input config_param :jpd_url, :string, default: "" config_param :username, :string, default: "" config_param :apikey, :string, default: "" - config_param :pos_file, :string, default: "" + config_param :pos_file, :string, default: "jfrog_siem.log.pos." + DateTime.now.strftime("%Y-%m-%d") config_param :batch_size, :integer, default: 25 config_param :thread_count, :integer, default: 5 config_param :wait_interval, :integer, default: 60 diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 890e30d..6ab7061 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -23,22 +23,10 @@ def violations(date_since) puts "Total violations count is #{total_violation_count}" if total_violation_count > 0 puts "Number of Violations in page #{page_number} are #{page_violation_count}" - last_limit_lines = get_last_limit_lines_from_pos_file(page_violation_count) - if is_pos_file_empty - resp['violations'].each do |v| - violations_channel << v - end + violations_channel = push_to_violations_channel(violations_channel, resp) else - resp['violations'].each do |v| - alreadyProcessed = check_if_violation_already_processed(v, last_limit_lines) - if !alreadyProcessed - puts "Not processed" - violations_channel << v - else - puts "Already processed" - end - end + violations_channel = push_unique_violations_to_violations_channel(violations_channel, resp, page_violation_count) end if page_violation_count == @batch_size page_number += 1 @@ -49,6 +37,27 @@ def violations(date_since) violations_channel end + def push_to_violations_channel(violations_channel, resp) + resp['violations'].each do |v| + violations_channel << v + end + violations_channel + end + + def push_unique_violations_to_violations_channel(violations_channel, resp, page_violation_count) + last_limit_lines = get_last_limit_lines_from_pos_file(page_violation_count) + resp['violations'].each do |v| + alreadyProcessed = check_if_violation_already_processed(v, last_limit_lines) + if !alreadyProcessed + puts "Not processed" + violations_channel << v + else + puts "Already processed" + end + end + violations_channel + end + def is_pos_file_empty() file_lines = File.foreach(@pos_file).count if file_lines == 0 @@ -73,24 +82,27 @@ def violation_details(violations_channel) violations_channel.each do |v| Concurrent::Promises.future(v) do |v7| pull_violation_details(v['violation_details_url']) - open(@pos_file, 'a') do |f| - created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") - f.puts [created_date, v['watch_name'], v['issue_id']].join(',') - end + write_to_pos_file(v) end end end + def write_to_pos_file(v) + open(@pos_file, 'a') do |f| + created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + f.puts [created_date, v['watch_name'], v['issue_id']].join(',') + end + end + def pull_violation_details(xray_violation_detail_url) begin - #puts "Pulling violation details for #{xray_violation_detail_url}" detailResp=get_xray_violations_detail(xray_violation_detail_url) time = Fluent::Engine.now detailResp_json = data_normalization(detailResp) #puts detailResp_json #router.emit(@tag, time, detailResp_json) rescue => e - puts "error1: #{e}" + puts "error: #{e}" raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" end end @@ -106,7 +118,7 @@ def get_xray_violations_detail(xray_violation_detail_url) when 200 return response.to_str else - puts "error2: #{response.to_json}" + puts "error: #{response.to_json}" raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." end end @@ -186,7 +198,7 @@ def get_xray_violations(xray_json) when 200 return response.to_str else - puts "error3: #{response.to_json}" + puts "error: #{response.to_json}" raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations. #{response.to_json}" end end From fdcab3838c1615839ec31543f64c90188e7cb151 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Tue, 13 Jul 2021 22:20:31 -0700 Subject: [PATCH 17/44] Removes unused config parameters - since we do not use threads anymore --- .../lib/fluent/plugin/in_jfrog_siem.rb | 9 --------- 1 file changed, 9 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index ddd8262..28d7949 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -33,7 +33,6 @@ class JfrogSiemInput < Fluent::Plugin::Input config_param :apikey, :string, default: "" config_param :pos_file, :string, default: "jfrog_siem.log.pos." + DateTime.now.strftime("%Y-%m-%d") config_param :batch_size, :integer, default: 25 - config_param :thread_count, :integer, default: 5 config_param :wait_interval, :integer, default: 60 config_param :from_date, :string, default: "" @@ -62,14 +61,6 @@ def configure(conf) raise Fluent::ConfigError, "Must define a position file to record last SIEM violation pulled." end - if @thread_count < 1 - raise Fluent::ConfigError, "Must define at least one thread to process violation details." - end - - if @thread_count > @batch_size - raise Fluent::ConfigError, "Violation detail url thread count exceeds batch size." - end - if @wait_interval < 1 raise Fluent::ConfigError, "Wait interval must be greater than 1 to wait between pulling new events." end From 674960b6d3c8ac2e0c9ea4bbb295e5aac2df981d Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Tue, 13 Jul 2021 22:21:17 -0700 Subject: [PATCH 18/44] Uses more idiomatic ruby with parsing of pos file by date - uses pos file by date search - removes last few lines calculations since the search is over a small file --- .../lib/fluent/plugin/xray.rb | 47 ++++++++----------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 6ab7061..53e18a4 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -23,10 +23,10 @@ def violations(date_since) puts "Total violations count is #{total_violation_count}" if total_violation_count > 0 puts "Number of Violations in page #{page_number} are #{page_violation_count}" - if is_pos_file_empty - violations_channel = push_to_violations_channel(violations_channel, resp) + if File.zero?(@pos_file) + violations_channel = push_to_violations_channel(violations_channel, resp['violations']) else - violations_channel = push_unique_violations_to_violations_channel(violations_channel, resp, page_violation_count) + violations_channel = push_unique_violations_to_violations_channel(violations_channel, resp['violations'], page_violation_count) end if page_violation_count == @batch_size page_number += 1 @@ -37,44 +37,35 @@ def violations(date_since) violations_channel end - def push_to_violations_channel(violations_channel, resp) - resp['violations'].each do |v| + def push_to_violations_channel(violations_channel, violations) + violations.each do |v| violations_channel << v end violations_channel end - def push_unique_violations_to_violations_channel(violations_channel, resp, page_violation_count) - last_limit_lines = get_last_limit_lines_from_pos_file(page_violation_count) - resp['violations'].each do |v| - alreadyProcessed = check_if_violation_already_processed(v, last_limit_lines) - if !alreadyProcessed - puts "Not processed" - violations_channel << v - else - puts "Already processed" + def push_unique_violations_to_violations_channel(violations_channel, violations, page_violation_count) + violations.each do |violation| + unless processed?(violation) + violations_channel << violation end end violations_channel end - def is_pos_file_empty() - file_lines = File.foreach(@pos_file).count - if file_lines == 0 - return true + def processed?(violation) + created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + violation_entry = [created_date, violation['watch_name'], violation['issue_id']].join(',') + processed = File.open "jfrog_siem_log_#{created_date}.pos" do |f| + f.find { |line| line =~ violation_entry } end - return false - end - - def check_if_violation_already_processed(v, last_limit_lines) - v_line = [DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ"), v['watch_name'], v['issue_id']].join(',') - return last_limit_lines.any? { |s| s.include?(v_line) } + return processed end - def get_last_limit_lines_from_pos_file(page_violation_count) - last_limit_lines = IO.readlines(@pos_file)[-page_violation_count..-1] - return last_limit_lines - end + # def get_last_limit_lines_from_pos_file(page_violation_count) + # last_limit_lines = IO.readlines(@pos_file)[-page_violation_count..-1] + # return last_limit_lines + # end def violation_details(violations_channel) # emit only violation details and not all From 7ada33425712d98afe82292d9a00389af99d1a80 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Wed, 14 Jul 2021 18:47:49 -0700 Subject: [PATCH 19/44] Adds specs for processed violations - removes specs covering old features that were deleted --- .../lib/fluent/plugin/xray.rb | 7 +--- .../spec/xray_spec.rb | 42 +++++++------------ 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 53e18a4..38220aa 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -57,16 +57,11 @@ def processed?(violation) created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ") violation_entry = [created_date, violation['watch_name'], violation['issue_id']].join(',') processed = File.open "jfrog_siem_log_#{created_date}.pos" do |f| - f.find { |line| line =~ violation_entry } + f.find { |line| line.include? violation_entry } end return processed end - # def get_last_limit_lines_from_pos_file(page_violation_count) - # last_limit_lines = IO.readlines(@pos_file)[-page_violation_count..-1] - # return last_limit_lines - # end - def violation_details(violations_channel) # emit only violation details and not all puts "violations details" diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index 1059f19..7ffc249 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -49,38 +49,26 @@ end end - describe "#violations_by_page" do - it "gets violations for for_date" do - xray = Xray.new("@jpd_url", @username, @apikey, @wait_interval, 5, @pos_file) + describe "#processed?" do + + let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } - rest_client = double("A Rest Client") - expect(RestClient::Request).to receive(:new).and_return rest_client - expect(rest_client).to receive(:execute).and_return('{"violations": [1, 2, 3, 4, 5]}') - - expect(JSON).to receive(:parse).and_return({'violations': [1, 2, 3, 4, 5]}) - - violations = xray.violations_by_page(Date.today, 1) - expect(violations).to eq ([1, 2, 3, 4, 5]) - end - end - - describe "#page_count" do - it "calculates page_count based on batch_size to account for last page smaller than batch_size" do - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, 5, @pos_file) - - expect(xray.page_count(24)).to be(5) - end - - it "calculates page_count based on batch_size to account for last page same as batch_size" do - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, 5, @pos_file) + it "returns false when a violation has not been processed" do + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) + + allow(File).to receive(:open).and_yield [] - expect(xray.page_count(20)).to be(4) + expect(xray.processed?(JSON.parse(violation.to_json))).to be_falsey end - it "returns elegantly when violations_count is 0" do - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, 5, @pos_file) + it "returns true when a violation was found in the pos file" do + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) + + matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',') + another_violation = [violation[:created], "watch2", "12345"].join(',') + allow(File).to receive(:open).and_yield [matching_violation, another_violation] - expect(xray.page_count(0)).to be(0) + expect(xray.processed?(JSON.parse(violation.to_json))).to be_truthy end end From 2f3367c0ff42c4e602444bd33a0c5a3879a99fd7 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Wed, 14 Jul 2021 19:02:01 -0700 Subject: [PATCH 20/44] Adds spec for writing to pos file --- .../lib/fluent/plugin/xray.rb | 5 +++-- .../fluent-plugin-jfrog-siem/spec/xray_spec.rb | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 38220aa..bc632bb 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -74,9 +74,10 @@ def violation_details(violations_channel) end def write_to_pos_file(v) - open(@pos_file, 'a') do |f| + File.open(@pos_file, 'a') do |f| + puts v created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") - f.puts [created_date, v['watch_name'], v['issue_id']].join(',') + f << [created_date, v['watch_name'], v['issue_id']].join(',') end end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index 7ffc249..505d9e5 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -50,7 +50,6 @@ end describe "#processed?" do - let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } it "returns false when a violation has not been processed" do @@ -73,4 +72,21 @@ end + describe "#write_to_pos_file" do + let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } + + it "returns false when a violation has not been processed" do + allow(File).to receive(:open).and_yield [] + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) + + result = [] + allow(File).to receive(:open).and_yield result + + xray.write_to_pos_file(JSON.parse(violation.to_json)) + + matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',') + expect(result.include? matching_violation).to be_truthy + end + end + end \ No newline at end of file From 714220945c368738824ba027790845504ba93f12 Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Fri, 16 Jul 2021 09:54:17 -0700 Subject: [PATCH 21/44] Adding pos_file changes --- .../lib/fluent/plugin/in_jfrog_siem.rb | 33 +++++++------- .../lib/fluent/plugin/xray.rb | 45 ++++++++++--------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 28d7949..b85ba00 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -31,7 +31,6 @@ class JfrogSiemInput < Fluent::Plugin::Input config_param :jpd_url, :string, default: "" config_param :username, :string, default: "" config_param :apikey, :string, default: "" - config_param :pos_file, :string, default: "jfrog_siem.log.pos." + DateTime.now.strftime("%Y-%m-%d") config_param :batch_size, :integer, default: 25 config_param :wait_interval, :integer, default: 60 config_param :from_date, :string, default: "" @@ -57,14 +56,14 @@ def configure(conf) raise Fluent::ConfigError, "Must define the API Key to use for authentication." end - if @pos_file == "" - raise Fluent::ConfigError, "Must define a position file to record last SIEM violation pulled." - end - if @wait_interval < 1 raise Fluent::ConfigError, "Wait interval must be greater than 1 to wait between pulling new events." end + if @from_date == "" + puts "From date not specified, so getting violations from current date" + end + end @@ -86,31 +85,31 @@ def shutdown def run call_home(@jpd_url) - last_created_date_string = get_last_item_create_date() - begin - last_created_date = DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ") - rescue - last_created_date = DateTime.now.strftime("%Y-%m-%dT%H:%M:%SZ") - end + last_created_date = get_last_item_create_date() if (@from_date != "") last_created_date = DateTime.parse(@from_date).strftime("%Y-%m-%dT%H:%M:%SZ") end date_since = last_created_date - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) violations_channel = xray.violations(date_since) xray.violation_details(violations_channel) - sleep 100 - end # pull the last item create date from the pos_file return created_date_string def get_last_item_create_date() - if(!(File.exist?(@pos_file))) - @pos_file = File.new(@pos_file, "w") + recent_pos_file = get_recent_pos_file() + if recent_pos_file != nil + last_created_date_string = IO.readlines(recent_pos_file).last + return DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ") + else + return DateTime.now.strftime("%Y-%m-%dT%H:%M:%SZ") end - return IO.readlines(@pos_file).last + end + + def get_recent_pos_file() + return Dir.glob("*.pos").sort[-1] end #call home functionality diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 53e18a4..7632ba0 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -3,13 +3,12 @@ require 'json' class Xray - def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file) + def initialize(jpd_url, username, api_key, wait_interval, batch_size) @jpd_url = jpd_url @username = username @api_key = api_key @wait_interval = wait_interval @batch_size = batch_size - @pos_file = pos_file end def violations(date_since) @@ -23,10 +22,14 @@ def violations(date_since) puts "Total violations count is #{total_violation_count}" if total_violation_count > 0 puts "Number of Violations in page #{page_number} are #{page_violation_count}" - if File.zero?(@pos_file) - violations_channel = push_to_violations_channel(violations_channel, resp['violations']) - else - violations_channel = push_unique_violations_to_violations_channel(violations_channel, resp['violations'], page_violation_count) + resp['violations'].each do |violation| + pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d") + temp_pos_file = "jfrog_siem_log_#{pos_file_date}.pos" + if(!(File.exist?(temp_pos_file))) + violations_channel = push_to_violations_channel(violations_channel, violation) + else + violations_channel = push_unique_violations_to_violations_channel(violations_channel, violation, temp_pos_file) + end end if page_violation_count == @batch_size page_number += 1 @@ -37,29 +40,27 @@ def violations(date_since) violations_channel end - def push_to_violations_channel(violations_channel, violations) - violations.each do |v| - violations_channel << v - end + def push_to_violations_channel(violations_channel, violation) + violations_channel << violation violations_channel end - def push_unique_violations_to_violations_channel(violations_channel, violations, page_violation_count) - violations.each do |violation| - unless processed?(violation) - violations_channel << violation - end + def push_unique_violations_to_violations_channel(violations_channel, violation, temp_pos_file) + unless processed?(violation, temp_pos_file) + violations_channel << violation end violations_channel end - def processed?(violation) + def processed?(violation, temp_pos_file) created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ") violation_entry = [created_date, violation['watch_name'], violation['issue_id']].join(',') - processed = File.open "jfrog_siem_log_#{created_date}.pos" do |f| - f.find { |line| line =~ violation_entry } + File.open(temp_pos_file).each_line do |line| + if (line.include?violation_entry) + return true + end end - return processed + return false end # def get_last_limit_lines_from_pos_file(page_violation_count) @@ -79,8 +80,10 @@ def violation_details(violations_channel) end def write_to_pos_file(v) - open(@pos_file, 'a') do |f| - created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + pos_file_date = DateTime.parse(v['created']).strftime("%Y-%m-%d") + current_pos_file = "jfrog_siem_log_#{pos_file_date}.pos" + open(current_pos_file, 'a') do |f| f.puts [created_date, v['watch_name'], v['issue_id']].join(',') end end From a5cf7eb4739489ebd2a4b867b610562933dbbc4c Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Mon, 19 Jul 2021 11:11:20 -0700 Subject: [PATCH 22/44] Changing param count --- .../fluent-plugin-jfrog-siem/spec/xray_spec.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index 505d9e5..17918d1 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -14,7 +14,7 @@ RSpec.describe Xray do describe "#violation_details" do it "creates a future for every violation" do - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) violations = Concurrent::Array.new (1..5).each do |i| @@ -52,22 +52,25 @@ describe "#processed?" do let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } + pos_file_date = DateTime.parse(Date.today.to_s).strftime("%Y-%m-%d") + temp_pos_file = "jfrog_siem_log_#{pos_file_date}.pos" + it "returns false when a violation has not been processed" do - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) allow(File).to receive(:open).and_yield [] - expect(xray.processed?(JSON.parse(violation.to_json))).to be_falsey + expect(xray.processed?(JSON.parse(violation.to_json), temp_pos_file)).to be_falsey end it "returns true when a violation was found in the pos file" do - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) - + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) + matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',') another_violation = [violation[:created], "watch2", "12345"].join(',') allow(File).to receive(:open).and_yield [matching_violation, another_violation] - expect(xray.processed?(JSON.parse(violation.to_json))).to be_truthy + expect(xray.processed?(JSON.parse(violation.to_json), temp_pos_file)).to be_truthy end end @@ -77,7 +80,7 @@ it "returns false when a violation has not been processed" do allow(File).to receive(:open).and_yield [] - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) result = [] allow(File).to receive(:open).and_yield result From c84ea09ebd28bb64d6d2a64e3493e6321b49bb86 Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Mon, 19 Jul 2021 12:26:28 -0700 Subject: [PATCH 23/44] xit for processed, file.open for write to file spec --- .../input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb | 4 ++-- .../plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 5177a75..58d66ac 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -78,8 +78,8 @@ def write_to_pos_file(v) created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") pos_file_date = DateTime.parse(v['created']).strftime("%Y-%m-%d") current_pos_file = "jfrog_siem_log_#{pos_file_date}.pos" - open(current_pos_file, 'a') do |f| - f.puts [created_date, v['watch_name'], v['issue_id']].join(',') + File.open(current_pos_file, 'a') do |f| + f << [created_date, v['watch_name'], v['issue_id']].join(',') end end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index 17918d1..592222b 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -55,7 +55,7 @@ pos_file_date = DateTime.parse(Date.today.to_s).strftime("%Y-%m-%d") temp_pos_file = "jfrog_siem_log_#{pos_file_date}.pos" - it "returns false when a violation has not been processed" do + xit "returns false when a violation has not been processed" do xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) allow(File).to receive(:open).and_yield [] @@ -63,7 +63,7 @@ expect(xray.processed?(JSON.parse(violation.to_json), temp_pos_file)).to be_falsey end - it "returns true when a violation was found in the pos file" do + xit "returns true when a violation was found in the pos file" do xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',') From 0257fbe7315876e096699129cec192ad0ece2fe3 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Mon, 19 Jul 2021 16:14:09 -0700 Subject: [PATCH 24/44] Fixed specs that were breaking because of using each_line - reverts to use f.find since it is more efficient with large files --- .../fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb | 8 +++----- .../input/fluent-plugin-jfrog-siem/spec/xray_spec.rb | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 58d66ac..4acdf2b 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -55,12 +55,10 @@ def push_unique_violations_to_violations_channel(violations_channel, violation, def processed?(violation, temp_pos_file) created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ") violation_entry = [created_date, violation['watch_name'], violation['issue_id']].join(',') - File.open(temp_pos_file).each_line do |line| - if (line.include?violation_entry) - return true - end + processed = File.open(temp_pos_file) do |f| + f.find { |line| line.include? violation_entry } end - return false + return processed end def violation_details(violations_channel) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index 592222b..17918d1 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -55,7 +55,7 @@ pos_file_date = DateTime.parse(Date.today.to_s).strftime("%Y-%m-%d") temp_pos_file = "jfrog_siem_log_#{pos_file_date}.pos" - xit "returns false when a violation has not been processed" do + it "returns false when a violation has not been processed" do xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) allow(File).to receive(:open).and_yield [] @@ -63,7 +63,7 @@ expect(xray.processed?(JSON.parse(violation.to_json), temp_pos_file)).to be_falsey end - xit "returns true when a violation was found in the pos file" do + it "returns true when a violation was found in the pos file" do xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',') From 2c227ca4e31c41f60b53f54ca2bf08920cacd204 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Mon, 19 Jul 2021 16:35:56 -0700 Subject: [PATCH 25/44] Moves logic for file parsing over to its own class - fixes all specs for position files --- .../lib/fluent/plugin/xray.rb | 33 ++-------- .../spec/xray_spec.rb | 61 ++++--------------- 2 files changed, 17 insertions(+), 77 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 4acdf2b..10ef70b 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -23,12 +23,10 @@ def violations(date_since) if total_violation_count > 0 puts "Number of Violations in page #{page_number} are #{page_violation_count}" resp['violations'].each do |violation| - pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d") - temp_pos_file = "jfrog_siem_log_#{pos_file_date}.pos" if(!(File.exist?(temp_pos_file))) violations_channel = push_to_violations_channel(violations_channel, violation) else - violations_channel = push_unique_violations_to_violations_channel(violations_channel, violation, temp_pos_file) + violations_channel = push_unique_violations_to_violations_channel(violations_channel, violation) end end if page_violation_count == @batch_size @@ -45,49 +43,28 @@ def push_to_violations_channel(violations_channel, violation) violations_channel end - def push_unique_violations_to_violations_channel(violations_channel, violation, temp_pos_file) - unless processed?(violation, temp_pos_file) + def push_unique_violations_to_violations_channel(violations_channel, violation) + unless PositionFile.new.processed?(violation) violations_channel << violation end violations_channel end - def processed?(violation, temp_pos_file) - created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ") - violation_entry = [created_date, violation['watch_name'], violation['issue_id']].join(',') - processed = File.open(temp_pos_file) do |f| - f.find { |line| line.include? violation_entry } - end - return processed - end - def violation_details(violations_channel) - # emit only violation details and not all - puts "violations details" violations_channel.each do |v| Concurrent::Promises.future(v) do |v7| pull_violation_details(v['violation_details_url']) - write_to_pos_file(v) + PositionFile.new.write(v) end end end - def write_to_pos_file(v) - created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") - pos_file_date = DateTime.parse(v['created']).strftime("%Y-%m-%d") - current_pos_file = "jfrog_siem_log_#{pos_file_date}.pos" - File.open(current_pos_file, 'a') do |f| - f << [created_date, v['watch_name'], v['issue_id']].join(',') - end - end - def pull_violation_details(xray_violation_detail_url) begin detailResp=get_xray_violations_detail(xray_violation_detail_url) time = Fluent::Engine.now detailResp_json = data_normalization(detailResp) - #puts detailResp_json - #router.emit(@tag, time, detailResp_json) + router.emit(@tag, time, detailResp_json) rescue => e puts "error: #{e}" raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index 17918d1..2f346ab 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -13,6 +13,10 @@ RSpec.describe Xray do describe "#violation_details" do + + let(:violation1){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } + let(:violation2){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch2", "issue_id": "55443"} } + it "creates a future for every violation" do xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) violations = Concurrent::Array.new @@ -30,66 +34,25 @@ xit "updates pos file for every violation" do pos_file = double('pos_file') - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, "pos_file.txt") + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) violations = Concurrent::Array.new - (1..5).each do |i| - violations << i - end - + violations << violation1 + violations << violation2 + datetime = double (DateTime) expect(datetime).to receive(:parse) promises = class_double("Concurrent::Promises").as_stubbed_const(:transfer_nested_constants => true) allow(promises).to receive(:future) { |&block| block.call } - expect(File).to receive(:open).with("pos_file.txt", "a").and_yield(pos_file) - expect(pos_file).to receive(:puts).exactly(5).times + fluent = class_double("Fluent::Engine").as_stubbed_const(:transfer_nested_constants => true) + expect(fluent).to receive(:now).and_return(DateTime.now) + pos_file = double (PositionFile) + expect(pos_file).to receive(:write).exactly(2).times xray.violation_details(violations) end end - describe "#processed?" do - let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } - - pos_file_date = DateTime.parse(Date.today.to_s).strftime("%Y-%m-%d") - temp_pos_file = "jfrog_siem_log_#{pos_file_date}.pos" - - it "returns false when a violation has not been processed" do - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) - - allow(File).to receive(:open).and_yield [] - - expect(xray.processed?(JSON.parse(violation.to_json), temp_pos_file)).to be_falsey - end - - it "returns true when a violation was found in the pos file" do - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) - - matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',') - another_violation = [violation[:created], "watch2", "12345"].join(',') - allow(File).to receive(:open).and_yield [matching_violation, another_violation] - - expect(xray.processed?(JSON.parse(violation.to_json), temp_pos_file)).to be_truthy - end - - end - - describe "#write_to_pos_file" do - let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } - - it "returns false when a violation has not been processed" do - allow(File).to receive(:open).and_yield [] - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) - - result = [] - allow(File).to receive(:open).and_yield result - - xray.write_to_pos_file(JSON.parse(violation.to_json)) - - matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',') - expect(result.include? matching_violation).to be_truthy - end - end end \ No newline at end of file From ea689feb121a477f19a0bbe3d978fa9b9a53fce1 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Tue, 20 Jul 2021 11:46:30 -0700 Subject: [PATCH 26/44] Adds Position file class. --- .../lib/fluent/plugin/position_file.rb | 23 ++++++++ .../spec/position_file_spec.rb | 55 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb create mode 100644 fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/position_file_spec.rb diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb new file mode 100644 index 0000000..67659ce --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb @@ -0,0 +1,23 @@ +class PositionFile + + def processed?(violation) + pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d") + pos_file = "jfrog_siem_log_#{pos_file_date}.pos" + created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + violation_entry = [created_date, violation['watch_name'], violation['issue_id']].join(',') + processed = File.open(pos_file) do |f| + f.find { |line| line.include? violation_entry } + end + return processed + end + + def write(v) + created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + pos_file_date = DateTime.parse(v['created']).strftime("%Y-%m-%d") + pos_file = "jfrog_siem_log_#{pos_file_date}.pos" + File.open(pos_file, 'a') do |f| + f << [created_date, v['watch_name'], v['issue_id']].join(',') + end + end + +end \ No newline at end of file diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/position_file_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/position_file_spec.rb new file mode 100644 index 0000000..6edce1e --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/position_file_spec.rb @@ -0,0 +1,55 @@ +[ + File.join(File.dirname(__FILE__), '..'), + File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'), + File.join(File.dirname(__FILE__), '..', 'spec'), +].each do |dir| + $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir) +end + +require 'position_file' +require 'date' +require 'rspec' + + +RSpec.describe PositionFile do + describe "#processed?" do + let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } + + pos_file_date = DateTime.parse(Date.today.to_s).strftime("%Y-%m-%d") + temp_pos_file = "jfrog_siem_log_#{pos_file_date}.pos" + + it "returns false when a violation has not been processed" do + pos_file = PositionFile.new + allow(File).to receive(:open).and_yield [] + + expect(pos_file.processed?(JSON.parse(violation.to_json))).to be_falsey + end + + it "returns true when a violation was found in the pos file" do + pos_file = PositionFile.new + + matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',') + another_violation = [violation[:created], "watch2", "12345"].join(',') + allow(File).to receive(:open).and_yield [matching_violation, another_violation] + + expect(pos_file.processed?(JSON.parse(violation.to_json))).to be_truthy + end + + end + + describe "#write" do + let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } + + it "returns false when a violation has not been processed" do + pos_file = PositionFile.new + + result = [] + allow(File).to receive(:open).and_yield result + + pos_file.write(JSON.parse(violation.to_json)) + + matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',') + expect(result.include? matching_violation).to be_truthy + end + end +end \ No newline at end of file From c2661c4a80d923314a6e66ce86c864a983c56992 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Tue, 20 Jul 2021 11:54:13 -0700 Subject: [PATCH 27/44] Adds missing reference to position file --- .../fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 10ef70b..5e01e21 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -23,10 +23,12 @@ def violations(date_since) if total_violation_count > 0 puts "Number of Violations in page #{page_number} are #{page_violation_count}" resp['violations'].each do |violation| - if(!(File.exist?(temp_pos_file))) - violations_channel = push_to_violations_channel(violations_channel, violation) - else + pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d") + pos_file = "jfrog_siem_log_#{pos_file_date}.pos" + if File.exist?(pos_file) violations_channel = push_unique_violations_to_violations_channel(violations_channel, violation) + else + violations_channel = push_to_violations_channel(violations_channel, violation) end end if page_violation_count == @batch_size From ac39c7bfed52e365f39040f9728d9348a55d82f3 Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Tue, 20 Jul 2021 13:02:38 -0700 Subject: [PATCH 28/44] Adding router.emit func, new line in file --- .../lib/fluent/plugin/in_jfrog_siem.rb | 120 ++++++++++++++++-- .../lib/fluent/plugin/position_file.rb | 2 +- .../lib/fluent/plugin/xray.rb | 99 +-------------- 3 files changed, 112 insertions(+), 109 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index b85ba00..3d4376b 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -17,6 +17,7 @@ require "date" require "uri" require "fluent/plugin/xray.rb" +require "fluent/plugin/position_file.rb" module Fluent module Plugin @@ -93,10 +94,25 @@ def run date_since = last_created_date xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) violations_channel = xray.violations(date_since) - xray.violation_details(violations_channel) + violation_details(violations_channel) sleep 100 end + #call home functionality + def call_home(jpd_url) + call_home_json = { "productId": "jfrogLogAnalytics/v0.5.1", "features": [ { "featureId": "Platform/Xray" }, { "featureId": "Channel/xrayeventsiem" } ] } + response = RestClient::Request.new( + :method => :post, + :url => jpd_url + "/artifactory/api/system/usage", + :payload => call_home_json.to_json, + :user => @username, + :password => @apikey, + :headers => { :accept => :json, :content_type => :json} + ).execute do |response, request, result| + puts "Posting call home information" + end + end + # pull the last item create date from the pos_file return created_date_string def get_last_item_create_date() recent_pos_file = get_recent_pos_file() @@ -112,20 +128,104 @@ def get_recent_pos_file() return Dir.glob("*.pos").sort[-1] end - #call home functionality - def call_home(jpd_url) - call_home_json = { "productId": "jfrogLogAnalytics/v0.5.1", "features": [ { "featureId": "Platform/Xray" }, { "featureId": "Channel/xrayeventsiem" } ] } + def violation_details(violations_channel) + violations_channel.each do |v| + Concurrent::Promises.future(v) do |v7| + pull_violation_details(v['violation_details_url']) + PositionFile.new.write(v) + end + end + end + + def pull_violation_details(xray_violation_detail_url) + begin + detailResp=get_xray_violations_detail(xray_violation_detail_url) + time = Fluent::Engine.now + detailResp_json = data_normalization(detailResp) + router.emit(@tag, time, detailResp_json) + rescue => e + puts "error: #{e}" + raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" + end + end + + def get_xray_violations_detail(xray_violation_detail_url) response = RestClient::Request.new( - :method => :post, - :url => jpd_url + "/artifactory/api/system/usage", - :payload => call_home_json.to_json, + :method => :get, + :url => xray_violation_detail_url, :user => @username, - :password => @apikey, - :headers => { :accept => :json, :content_type => :json} + :password => @apikey ).execute do |response, request, result| - puts "Posting call home information" + case response.code + when 200 + return response.to_str + else + puts "error: #{response.to_json}" + raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." + end end end + + def data_normalization(detailResp) + detailResp_json = JSON.parse(detailResp) + cve = [] + cvss_v2_list = [] + cvss_v3_list = [] + policy_list = [] + rule_list = [] + impacted_artifact_url_list = [] + if detailResp_json.key?('properties') + properties = detailResp_json['properties'] + for index in 0..properties.length-1 do + if properties[index].key?('cve') + cve.push(properties[index]['cve']) + end + if properties[index].key?('cvss_v2') + cvss_v2_list.push(properties[index]['cvss_v2']) + end + if properties[index].key?('cvss_v3') + cvss_v3_list.push(properties[index]['cvss_v3']) + end + end + + detailResp_json["cve"] = cve.sort.reverse[0] + cvss_v2 = cvss_v2_list.sort.reverse[0] + cvss_v3 = cvss_v3_list.sort.reverse[0] + if !cvss_v3.nil? + cvss = cvss_v3 + elsif !cvss_v2.nil? + cvss = cvss_v2 + end + cvss_score = cvss[0..2] + cvss_version = cvss.split(':')[1][0..2] + detailResp_json["cvss_score"] = cvss_score + detailResp_json["cvss_version"] = cvss_version + end + + if detailResp_json.key?('matched_policies') + matched_policies = detailResp_json['matched_policies'] + for index in 0..matched_policies.length-1 do + if matched_policies[index].key?('policy') + policy_list.push(matched_policies[index]['policy']) + end + if matched_policies[index].key?('rule') + rule_list.push(matched_policies[index]['rule']) + end + end + detailResp_json['policies'] = policy_list + detailResp_json['rules'] = rule_list + end + + impacted_artifacts = detailResp_json['impacted_artifacts'] + for impacted_artifact in impacted_artifacts do + matchdata = impacted_artifact.match /default\/(?[^\/]*)\/(?.*)/ + impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " " + impacted_artifact_url_list.append(impacted_artifact_url) + end + detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list + return detailResp_json + end + end end end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb index 67659ce..5fb8262 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb @@ -17,7 +17,7 @@ def write(v) pos_file = "jfrog_siem_log_#{pos_file_date}.pos" File.open(pos_file, 'a') do |f| f << [created_date, v['watch_name'], v['issue_id']].join(',') + f << "\n" end end - end \ No newline at end of file diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 5e01e21..e633552 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -1,6 +1,7 @@ require 'concurrent' require 'concurrent-edge' require 'json' +require "fluent/plugin/position_file.rb" class Xray def initialize(jpd_url, username, api_key, wait_interval, batch_size) @@ -52,104 +53,6 @@ def push_unique_violations_to_violations_channel(violations_channel, violation) violations_channel end - def violation_details(violations_channel) - violations_channel.each do |v| - Concurrent::Promises.future(v) do |v7| - pull_violation_details(v['violation_details_url']) - PositionFile.new.write(v) - end - end - end - - def pull_violation_details(xray_violation_detail_url) - begin - detailResp=get_xray_violations_detail(xray_violation_detail_url) - time = Fluent::Engine.now - detailResp_json = data_normalization(detailResp) - router.emit(@tag, time, detailResp_json) - rescue => e - puts "error: #{e}" - raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" - end - end - - def get_xray_violations_detail(xray_violation_detail_url) - response = RestClient::Request.new( - :method => :get, - :url => xray_violation_detail_url, - :user => @username, - :password => @api_key - ).execute do |response, request, result| - case response.code - when 200 - return response.to_str - else - puts "error: #{response.to_json}" - raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." - end - end - end - - def data_normalization(detailResp) - detailResp_json = JSON.parse(detailResp) - cve = [] - cvss_v2_list = [] - cvss_v3_list = [] - policy_list = [] - rule_list = [] - impacted_artifact_url_list = [] - if detailResp_json.key?('properties') - properties = detailResp_json['properties'] - for index in 0..properties.length-1 do - if properties[index].key?('cve') - cve.push(properties[index]['cve']) - end - if properties[index].key?('cvss_v2') - cvss_v2_list.push(properties[index]['cvss_v2']) - end - if properties[index].key?('cvss_v3') - cvss_v3_list.push(properties[index]['cvss_v3']) - end - end - - detailResp_json["cve"] = cve.sort.reverse[0] - cvss_v2 = cvss_v2_list.sort.reverse[0] - cvss_v3 = cvss_v3_list.sort.reverse[0] - if !cvss_v3.nil? - cvss = cvss_v3 - elsif !cvss_v2.nil? - cvss = cvss_v2 - end - cvss_score = cvss[0..2] - cvss_version = cvss.split(':')[1][0..2] - detailResp_json["cvss_score"] = cvss_score - detailResp_json["cvss_version"] = cvss_version - end - - if detailResp_json.key?('matched_policies') - matched_policies = detailResp_json['matched_policies'] - for index in 0..matched_policies.length-1 do - if matched_policies[index].key?('policy') - policy_list.push(matched_policies[index]['policy']) - end - if matched_policies[index].key?('rule') - rule_list.push(matched_policies[index]['rule']) - end - end - detailResp_json['policies'] = policy_list - detailResp_json['rules'] = rule_list - end - - impacted_artifacts = detailResp_json['impacted_artifacts'] - for impacted_artifact in impacted_artifacts do - matchdata = impacted_artifact.match /default\/(?[^\/]*)\/(?.*)/ - impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " " - impacted_artifact_url_list.append(impacted_artifact_url) - end - detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list - return detailResp_json - end - private def get_xray_violations(xray_json) response = RestClient::Request.new( From e18ed5a3370e32808958ba40de3392f46252fb02 Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Tue, 20 Jul 2021 13:32:30 -0700 Subject: [PATCH 29/44] Adding pos_file_path changes --- .../fluent-plugin-jfrog-siem.gemspec | 2 +- .../lib/fluent/plugin/in_jfrog_siem.rb | 7 +++++-- .../lib/fluent/plugin/position_file.rb | 4 ++-- .../fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb | 2 +- .../test/plugin/test_in_jfrog_siem.rb | 10 +++++----- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec b/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec index 8b14391..0ce98ed 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec @@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| spec.name = "fluent-plugin-jfrog-siem" - spec.version = "1.0.0" + spec.version = "2.0.0" spec.authors = ["John Peterson", "Mahitha Byreddy"] spec.email = ["johnp@jfrog.com", "mahithab@jfrog.com"] diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 3d4376b..5bc02b4 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -35,6 +35,7 @@ class JfrogSiemInput < Fluent::Plugin::Input config_param :batch_size, :integer, default: 25 config_param :wait_interval, :integer, default: 60 config_param :from_date, :string, default: "" + config_param :pos_file_path, :string, default: "" # `configure` is called before `start`. # 'conf' is a `Hash` that includes the configuration parameters. @@ -62,7 +63,7 @@ def configure(conf) end if @from_date == "" - puts "From date not specified, so getting violations from current date" + puts "From date not specified, so getting violations from current date if pos_file doesn't exist" end end @@ -92,6 +93,7 @@ def run last_created_date = DateTime.parse(@from_date).strftime("%Y-%m-%dT%H:%M:%SZ") end date_since = last_created_date + puts date_since xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) violations_channel = xray.violations(date_since) violation_details(violations_channel) @@ -125,7 +127,8 @@ def get_last_item_create_date() end def get_recent_pos_file() - return Dir.glob("*.pos").sort[-1] + pos_file = @pos_file_path + "*.siem.pos" + return Dir.glob(pos_file).sort[-1] end def violation_details(violations_channel) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb index 5fb8262..e1d1d45 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb @@ -2,7 +2,7 @@ class PositionFile def processed?(violation) pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d") - pos_file = "jfrog_siem_log_#{pos_file_date}.pos" + pos_file = "jfrog_siem_log_#{pos_file_date}.siem.pos" created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ") violation_entry = [created_date, violation['watch_name'], violation['issue_id']].join(',') processed = File.open(pos_file) do |f| @@ -14,7 +14,7 @@ def processed?(violation) def write(v) created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") pos_file_date = DateTime.parse(v['created']).strftime("%Y-%m-%d") - pos_file = "jfrog_siem_log_#{pos_file_date}.pos" + pos_file = "jfrog_siem_log_#{pos_file_date}.siem.pos" File.open(pos_file, 'a') do |f| f << [created_date, v['watch_name'], v['issue_id']].join(',') f << "\n" diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index e633552..a877f38 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -25,7 +25,7 @@ def violations(date_since) puts "Number of Violations in page #{page_number} are #{page_violation_count}" resp['violations'].each do |violation| pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d") - pos_file = "jfrog_siem_log_#{pos_file_date}.pos" + pos_file = "jfrog_siem_log_#{pos_file_date}.siem.pos" if File.exist?(pos_file) violations_channel = push_unique_violations_to_violations_channel(violations_channel, violation) else diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb index 323ce0e..a20fe4f 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb @@ -21,13 +21,13 @@ class JfrogSiemInputTest < Test::Unit::TestCase # ] CONFIG = %[ - tag "sudhindra-xray-rt.test_tag" - jpd_url "https://sudhindra-xray-rt.jfrog.tech/" + tag "jfrog.xray.siem.vulnerabilities" + jpd_url "JPDURL" username "admin" - apikey "AKCp8jQd1zP4oKv43SNgewrNwikd1iAQznfhSfx3T249eVMkGnJnSjCpNsuv8vtHWChKLfJ1w" - pos_file "test_pos.txt" + apikey "APIKEY" + pos_file_path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/" wait_interval 10 - start_date "2021-06-15" + from_date "2016-01-01" batch_size 25 ] From cc4e72d9f52d1968e2f60d9e3890db618d27352c Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Tue, 20 Jul 2021 14:05:22 -0700 Subject: [PATCH 30/44] Ading pos_file_path addition to existing name --- .../fluent-plugin-jfrog-siem.gemspec | 3 ++- .../lib/fluent/plugin/in_jfrog_siem.rb | 6 +++--- .../lib/fluent/plugin/position_file.rb | 8 ++++++-- .../fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb | 7 ++++--- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec b/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec index 0ce98ed..7217dbc 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec @@ -27,7 +27,8 @@ Gem::Specification.new do |spec| spec.add_development_dependency "concurrent-ruby", "~> 1.1.8" spec.add_development_dependency "concurrent-ruby-edge", '>= 0' spec.add_development_dependency 'rspec', '~> 3.10.0' - + + spec.add_runtime_dependency "rest-client", "~> 2.0" spec.add_runtime_dependency "concurrent-ruby", "~> 1.1.8" spec.add_runtime_dependency "concurrent-ruby-edge", '>= 0' spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"] diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 5bc02b4..067643e 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -93,8 +93,8 @@ def run last_created_date = DateTime.parse(@from_date).strftime("%Y-%m-%dT%H:%M:%SZ") end date_since = last_created_date - puts date_since - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) + puts "Getting queries from #{date_since}" + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path) violations_channel = xray.violations(date_since) violation_details(violations_channel) sleep 100 @@ -135,7 +135,7 @@ def violation_details(violations_channel) violations_channel.each do |v| Concurrent::Promises.future(v) do |v7| pull_violation_details(v['violation_details_url']) - PositionFile.new.write(v) + PositionFile.new(@pos_file_path).write(v) end end end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb index e1d1d45..68a125d 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb @@ -1,8 +1,12 @@ class PositionFile + def initialize(pos_file_path) + @pos_file_path = pos_file_path + end + def processed?(violation) pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d") - pos_file = "jfrog_siem_log_#{pos_file_date}.siem.pos" + pos_file = @pos_file_path + "jfrog_siem_log_#{pos_file_date}.siem.pos" created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ") violation_entry = [created_date, violation['watch_name'], violation['issue_id']].join(',') processed = File.open(pos_file) do |f| @@ -14,7 +18,7 @@ def processed?(violation) def write(v) created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") pos_file_date = DateTime.parse(v['created']).strftime("%Y-%m-%d") - pos_file = "jfrog_siem_log_#{pos_file_date}.siem.pos" + pos_file = @pos_file_path + "jfrog_siem_log_#{pos_file_date}.siem.pos" File.open(pos_file, 'a') do |f| f << [created_date, v['watch_name'], v['issue_id']].join(',') f << "\n" diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index a877f38..806218a 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -4,12 +4,13 @@ require "fluent/plugin/position_file.rb" class Xray - def initialize(jpd_url, username, api_key, wait_interval, batch_size) + def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file_path) @jpd_url = jpd_url @username = username @api_key = api_key @wait_interval = wait_interval @batch_size = batch_size + @pos_file_path = pos_file_path end def violations(date_since) @@ -25,7 +26,7 @@ def violations(date_since) puts "Number of Violations in page #{page_number} are #{page_violation_count}" resp['violations'].each do |violation| pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d") - pos_file = "jfrog_siem_log_#{pos_file_date}.siem.pos" + pos_file = @pos_file_path + "jfrog_siem_log_#{pos_file_date}.siem.pos" if File.exist?(pos_file) violations_channel = push_unique_violations_to_violations_channel(violations_channel, violation) else @@ -47,7 +48,7 @@ def push_to_violations_channel(violations_channel, violation) end def push_unique_violations_to_violations_channel(violations_channel, violation) - unless PositionFile.new.processed?(violation) + unless PositionFile.new(@pos_file_path).processed?(violation) violations_channel << violation end violations_channel From a69f11c93d50e5d5c39e9b3046d698a9b504b9ae Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Wed, 21 Jul 2021 14:08:23 -0700 Subject: [PATCH 31/44] Comments on call_home for testing right now --- .../lib/fluent/plugin/in_jfrog_siem.rb | 108 +----------------- .../lib/fluent/plugin/xray.rb | 101 +++++++++++++++- .../spec/xray_spec.rb | 4 +- 3 files changed, 107 insertions(+), 106 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 067643e..8a37a82 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -16,8 +16,8 @@ require "rest-client" require "date" require "uri" -require "fluent/plugin/xray.rb" -require "fluent/plugin/position_file.rb" +require "fluent/plugin/xray" +require "fluent/plugin/position_file" module Fluent module Plugin @@ -85,7 +85,7 @@ def shutdown def run - call_home(@jpd_url) + # call_home(@jpd_url) last_created_date = get_last_item_create_date() @@ -94,9 +94,9 @@ def run end date_since = last_created_date puts "Getting queries from #{date_since}" - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, router) violations_channel = xray.violations(date_since) - violation_details(violations_channel) + xray.violation_details(violations_channel) sleep 100 end @@ -131,104 +131,6 @@ def get_recent_pos_file() return Dir.glob(pos_file).sort[-1] end - def violation_details(violations_channel) - violations_channel.each do |v| - Concurrent::Promises.future(v) do |v7| - pull_violation_details(v['violation_details_url']) - PositionFile.new(@pos_file_path).write(v) - end - end - end - - def pull_violation_details(xray_violation_detail_url) - begin - detailResp=get_xray_violations_detail(xray_violation_detail_url) - time = Fluent::Engine.now - detailResp_json = data_normalization(detailResp) - router.emit(@tag, time, detailResp_json) - rescue => e - puts "error: #{e}" - raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" - end - end - - def get_xray_violations_detail(xray_violation_detail_url) - response = RestClient::Request.new( - :method => :get, - :url => xray_violation_detail_url, - :user => @username, - :password => @apikey - ).execute do |response, request, result| - case response.code - when 200 - return response.to_str - else - puts "error: #{response.to_json}" - raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." - end - end - end - - def data_normalization(detailResp) - detailResp_json = JSON.parse(detailResp) - cve = [] - cvss_v2_list = [] - cvss_v3_list = [] - policy_list = [] - rule_list = [] - impacted_artifact_url_list = [] - if detailResp_json.key?('properties') - properties = detailResp_json['properties'] - for index in 0..properties.length-1 do - if properties[index].key?('cve') - cve.push(properties[index]['cve']) - end - if properties[index].key?('cvss_v2') - cvss_v2_list.push(properties[index]['cvss_v2']) - end - if properties[index].key?('cvss_v3') - cvss_v3_list.push(properties[index]['cvss_v3']) - end - end - - detailResp_json["cve"] = cve.sort.reverse[0] - cvss_v2 = cvss_v2_list.sort.reverse[0] - cvss_v3 = cvss_v3_list.sort.reverse[0] - if !cvss_v3.nil? - cvss = cvss_v3 - elsif !cvss_v2.nil? - cvss = cvss_v2 - end - cvss_score = cvss[0..2] - cvss_version = cvss.split(':')[1][0..2] - detailResp_json["cvss_score"] = cvss_score - detailResp_json["cvss_version"] = cvss_version - end - - if detailResp_json.key?('matched_policies') - matched_policies = detailResp_json['matched_policies'] - for index in 0..matched_policies.length-1 do - if matched_policies[index].key?('policy') - policy_list.push(matched_policies[index]['policy']) - end - if matched_policies[index].key?('rule') - rule_list.push(matched_policies[index]['rule']) - end - end - detailResp_json['policies'] = policy_list - detailResp_json['rules'] = rule_list - end - - impacted_artifacts = detailResp_json['impacted_artifacts'] - for impacted_artifact in impacted_artifacts do - matchdata = impacted_artifact.match /default\/(?[^\/]*)\/(?.*)/ - impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " " - impacted_artifact_url_list.append(impacted_artifact_url) - end - detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list - return detailResp_json - end - end end end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 806218a..e9a15f1 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -4,13 +4,14 @@ require "fluent/plugin/position_file.rb" class Xray - def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file_path) + def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file_path, router) @jpd_url = jpd_url @username = username @api_key = api_key @wait_interval = wait_interval @batch_size = batch_size @pos_file_path = pos_file_path + @router = router end def violations(date_since) @@ -54,6 +55,104 @@ def push_unique_violations_to_violations_channel(violations_channel, violation) violations_channel end + def violation_details(violations_channel) + violations_channel.each do |v| + Concurrent::Promises.future(v) do |v7| + pull_violation_details(v['violation_details_url']) + PositionFile.new.write(v) + end + end + end + + def pull_violation_details(xray_violation_detail_url) + begin + detailResp = get_xray_violations_detail(xray_violation_detail_url) + time = Fluent::Engine.now + detailResp_json = data_normalization(detailResp) + router.emit(@tag, time, detailResp_json) + rescue => e + puts "error: #{e}" + raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" + end + end + + def get_xray_violations_detail(xray_violation_detail_url) + response = RestClient::Request.new( + :method => :get, + :url => xray_violation_detail_url, + :user => @username, + :password => @apikey + ).execute do |response, request, result| + case response.code + when 200 + return response.to_str + else + puts "error: #{response.to_json}" + raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." + end + end + end + + def data_normalization(detailResp) + detailResp_json = JSON.parse(detailResp) + cve = [] + cvss_v2_list = [] + cvss_v3_list = [] + policy_list = [] + rule_list = [] + impacted_artifact_url_list = [] + if detailResp_json.key?('properties') + properties = detailResp_json['properties'] + for index in 0..properties.length-1 do + if properties[index].key?('cve') + cve.push(properties[index]['cve']) + end + if properties[index].key?('cvss_v2') + cvss_v2_list.push(properties[index]['cvss_v2']) + end + if properties[index].key?('cvss_v3') + cvss_v3_list.push(properties[index]['cvss_v3']) + end + end + + detailResp_json["cve"] = cve.sort.reverse[0] + cvss_v2 = cvss_v2_list.sort.reverse[0] + cvss_v3 = cvss_v3_list.sort.reverse[0] + if !cvss_v3.nil? + cvss = cvss_v3 + elsif !cvss_v2.nil? + cvss = cvss_v2 + end + cvss_score = cvss[0..2] + cvss_version = cvss.split(':')[1][0..2] + detailResp_json["cvss_score"] = cvss_score + detailResp_json["cvss_version"] = cvss_version + end + + if detailResp_json.key?('matched_policies') + matched_policies = detailResp_json['matched_policies'] + for index in 0..matched_policies.length-1 do + if matched_policies[index].key?('policy') + policy_list.push(matched_policies[index]['policy']) + end + if matched_policies[index].key?('rule') + rule_list.push(matched_policies[index]['rule']) + end + end + detailResp_json['policies'] = policy_list + detailResp_json['rules'] = rule_list + end + + impacted_artifacts = detailResp_json['impacted_artifacts'] + for impacted_artifact in impacted_artifacts do + matchdata = impacted_artifact.match /default\/(?[^\/]*)\/(?.*)/ + impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " " + impacted_artifact_url_list.append(impacted_artifact_url) + end + detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list + return detailResp_json + end + private def get_xray_violations(xray_json) response = RestClient::Request.new( diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index 2f346ab..fc1c9fa 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -18,7 +18,7 @@ let(:violation2){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch2", "issue_id": "55443"} } it "creates a future for every violation" do - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @router) violations = Concurrent::Array.new (1..5).each do |i| @@ -34,7 +34,7 @@ xit "updates pos file for every violation" do pos_file = double('pos_file') - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @router) violations = Concurrent::Array.new violations << violation1 From 1d9d85dcfadee35fe9e6700361c2ce4efc022a24 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Wed, 21 Jul 2021 14:09:36 -0700 Subject: [PATCH 32/44] Moves logic into delegates as much as possible - passes router so that xray class can router.emit - comments out the call out to verify the write action on file for now - reduces complexiy and duplication in position file handling - reduces number of if statements. - rspecs are still passing. --- .../lib/fluent/plugin/in_jfrog_siem.rb | 2 +- .../lib/fluent/plugin/position_file.rb | 32 +++++++++------- .../lib/fluent/plugin/xray.rb | 37 +++++++------------ .../spec/position_file_spec.rb | 7 ++-- .../spec/xray_spec.rb | 2 +- .../test/plugin/test_in_jfrog_siem.rb | 24 ++++++++---- 6 files changed, 55 insertions(+), 49 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 8a37a82..1ede215 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -128,7 +128,7 @@ def get_last_item_create_date() def get_recent_pos_file() pos_file = @pos_file_path + "*.siem.pos" - return Dir.glob(pos_file).sort[-1] + return Dir.glob(pos_file).sort.last end end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb index 68a125d..ad1588f 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb @@ -5,23 +5,27 @@ def initialize(pos_file_path) end def processed?(violation) - pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d") - pos_file = @pos_file_path + "jfrog_siem_log_#{pos_file_date}.siem.pos" - created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ") - violation_entry = [created_date, violation['watch_name'], violation['issue_id']].join(',') - processed = File.open(pos_file) do |f| - f.find { |line| line.include? violation_entry } - end - return processed + File.exist?(pos_file_name(violation)) && found?(violation) end - def write(v) - created_date = DateTime.parse(v['created']).strftime("%Y-%m-%dT%H:%M:%SZ") - pos_file_date = DateTime.parse(v['created']).strftime("%Y-%m-%d") - pos_file = @pos_file_path + "jfrog_siem_log_#{pos_file_date}.siem.pos" - File.open(pos_file, 'a') do |f| - f << [created_date, v['watch_name'], v['issue_id']].join(',') + def found?(violation) + return File.open(pos_file_name(violation)) { |f| f.find { |line| line.include? violation_entry(violation) } } + end + + def write(violation) + File.open(pos_file_name(violation), 'a') do |f| + f << violation_entry(violation) f << "\n" end end + + def violation_entry(violation) + created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + [created_date, violation['watch_name'], violation['issue_id']].join(',') + end + + def pos_file_name(violation) + pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d") + @pos_file_path + "jfrog_siem_log_#{pos_file_date}.siem.pos" + end end \ No newline at end of file diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index e9a15f1..6767518 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -1,7 +1,7 @@ require 'concurrent' require 'concurrent-edge' require 'json' -require "fluent/plugin/position_file.rb" +require "fluent/plugin/position_file" class Xray def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file_path, router) @@ -16,8 +16,11 @@ def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file_p def violations(date_since) violations_channel = Concurrent::Channel.new(capacity: @batch_size) - page_number = 1 + page_number_channel = Concurrent::Channel.new(capacity: 1) + page_number_channel << 1 timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do + page_number = page_number_channel.take + puts page_number xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } resp = JSON.parse(get_xray_violations(xray_json)) total_violation_count = resp['total_violations'] @@ -26,40 +29,28 @@ def violations(date_since) if total_violation_count > 0 puts "Number of Violations in page #{page_number} are #{page_violation_count}" resp['violations'].each do |violation| - pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d") - pos_file = @pos_file_path + "jfrog_siem_log_#{pos_file_date}.siem.pos" - if File.exist?(pos_file) - violations_channel = push_unique_violations_to_violations_channel(violations_channel, violation) - else - violations_channel = push_to_violations_channel(violations_channel, violation) + pos_file = PositionFile.new(@pos_file_path) + unless pos_file.processed?(violation) + violations_channel << violation end end if page_violation_count == @batch_size page_number += 1 + page_number_channel << page_number end end end timer_task.execute - violations_channel - end - - def push_to_violations_channel(violations_channel, violation) - violations_channel << violation - violations_channel - end - def push_unique_violations_to_violations_channel(violations_channel, violation) - unless PositionFile.new(@pos_file_path).processed?(violation) - violations_channel << violation - end violations_channel end def violation_details(violations_channel) violations_channel.each do |v| - Concurrent::Promises.future(v) do |v7| + Concurrent::Promises.future(v) do |v| pull_violation_details(v['violation_details_url']) - PositionFile.new.write(v) + pos_file = PositionFile.new(@pos_file_path) + pos_file.write(v) end end end @@ -69,7 +60,7 @@ def pull_violation_details(xray_violation_detail_url) detailResp = get_xray_violations_detail(xray_violation_detail_url) time = Fluent::Engine.now detailResp_json = data_normalization(detailResp) - router.emit(@tag, time, detailResp_json) + # @router.emit(@tag, time, detailResp_json) rescue => e puts "error: #{e}" raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" @@ -81,7 +72,7 @@ def get_xray_violations_detail(xray_violation_detail_url) :method => :get, :url => xray_violation_detail_url, :user => @username, - :password => @apikey + :password => @api_key ).execute do |response, request, result| case response.code when 200 diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/position_file_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/position_file_spec.rb index 6edce1e..d524b1f 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/position_file_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/position_file_spec.rb @@ -19,17 +19,18 @@ temp_pos_file = "jfrog_siem_log_#{pos_file_date}.pos" it "returns false when a violation has not been processed" do - pos_file = PositionFile.new + pos_file = PositionFile.new(`pwd`) allow(File).to receive(:open).and_yield [] expect(pos_file.processed?(JSON.parse(violation.to_json))).to be_falsey end it "returns true when a violation was found in the pos file" do - pos_file = PositionFile.new + pos_file = PositionFile.new(`pwd`) matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',') another_violation = [violation[:created], "watch2", "12345"].join(',') + allow(File).to receive(:exist?).and_return true allow(File).to receive(:open).and_yield [matching_violation, another_violation] expect(pos_file.processed?(JSON.parse(violation.to_json))).to be_truthy @@ -41,7 +42,7 @@ let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } it "returns false when a violation has not been processed" do - pos_file = PositionFile.new + pos_file = PositionFile.new(`pwd`) result = [] allow(File).to receive(:open).and_yield result diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index fc1c9fa..e3b669d 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -18,7 +18,7 @@ let(:violation2){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch2", "issue_id": "55443"} } it "creates a future for every violation" do - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @router) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router) violations = Concurrent::Array.new (1..5).each do |i| diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb index a20fe4f..27227ee 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb @@ -21,16 +21,26 @@ class JfrogSiemInputTest < Test::Unit::TestCase # ] CONFIG = %[ - tag "jfrog.xray.siem.vulnerabilities" - jpd_url "JPDURL" + tag "sudhindra-xray-rt.test_tag" + jpd_url "https://sudhindra-xray-rt.jfrog.tech/" username "admin" - apikey "APIKEY" - pos_file_path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/" - wait_interval 10 - from_date "2016-01-01" - batch_size 25 + apikey "AKCp8jQd1zP4oKv43SNgewrNwikd1iAQznfhSfx3T249eVMkGnJnSjCpNsuv8vtHWChKLfJ1w" + wait_interval 1 + from_date "2021-06-15" + pos_file_path "/Users/sudhindrar/ptreng/log-analytics/fluentd/plugins/input/fluent-plugin-jfrog-siem/" ] + # CONFIG = %[ + # tag "jfrog.xray.siem.vulnerabilities" + # jpd_url "JPDURL" + # username "admin" + # apikey "APIKEY" + # pos_file_path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/" + # wait_interval 10 + # from_date "2016-01-01" + # batch_size 25 + # ] + private def create_driver(conf = CONFIG) From 145b01c2b692c0cca631c13a084b2b7c0ba5a000 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Wed, 21 Jul 2021 14:34:35 -0700 Subject: [PATCH 33/44] Moves methods to private in pos_file - uncomments router.emit --- .../lib/fluent/plugin/position_file.rb | 9 +++++---- .../fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb | 3 +-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb index ad1588f..1c7ac99 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb @@ -8,10 +8,6 @@ def processed?(violation) File.exist?(pos_file_name(violation)) && found?(violation) end - def found?(violation) - return File.open(pos_file_name(violation)) { |f| f.find { |line| line.include? violation_entry(violation) } } - end - def write(violation) File.open(pos_file_name(violation), 'a') do |f| f << violation_entry(violation) @@ -19,6 +15,11 @@ def write(violation) end end + private + def found?(violation) + return File.open(pos_file_name(violation)) { |f| f.find { |line| line.include? violation_entry(violation) } } + end + def violation_entry(violation) created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ") [created_date, violation['watch_name'], violation['issue_id']].join(',') diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 6767518..cfdffe2 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -20,7 +20,6 @@ def violations(date_since) page_number_channel << 1 timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do page_number = page_number_channel.take - puts page_number xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } resp = JSON.parse(get_xray_violations(xray_json)) total_violation_count = resp['total_violations'] @@ -60,7 +59,7 @@ def pull_violation_details(xray_violation_detail_url) detailResp = get_xray_violations_detail(xray_violation_detail_url) time = Fluent::Engine.now detailResp_json = data_normalization(detailResp) - # @router.emit(@tag, time, detailResp_json) + @router.emit(@tag, time, detailResp_json) rescue => e puts "error: #{e}" raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" From 209118e044a584b054ef719203075b02fa56d182 Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Wed, 21 Jul 2021 16:27:17 -0700 Subject: [PATCH 34/44] Reverting page_num_channel changs for live data --- .../input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index cfdffe2..59e5c7b 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -16,10 +16,8 @@ def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file_p def violations(date_since) violations_channel = Concurrent::Channel.new(capacity: @batch_size) - page_number_channel = Concurrent::Channel.new(capacity: 1) - page_number_channel << 1 + page_number = 1 timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do - page_number = page_number_channel.take xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } resp = JSON.parse(get_xray_violations(xray_json)) total_violation_count = resp['total_violations'] @@ -35,7 +33,6 @@ def violations(date_since) end if page_violation_count == @batch_size page_number += 1 - page_number_channel << page_number end end end From 236939c98012f368912eb18044beac72d7975529 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Fri, 23 Jul 2021 15:01:20 -0700 Subject: [PATCH 35/44] Removes JSON parsing from receiver - moves JSON parsing in the lower level method - fixes test for integration flow and writing to file, also emits to router -(concurrent ruby stubs seem to be not working with exactly calls so unable to test exact number of calls - but limited to testing the flow) --- .../lib/fluent/plugin/xray.rb | 19 ++++---- .../spec/xray_spec.rb | 48 ++++++++++++------- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 59e5c7b..ae4c1b9 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -19,7 +19,7 @@ def violations(date_since) page_number = 1 timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } - resp = JSON.parse(get_xray_violations(xray_json)) + resp = get_violations(xray_json) total_violation_count = resp['total_violations'] page_violation_count = resp['violations'].length puts "Total violations count is #{total_violation_count}" @@ -53,9 +53,8 @@ def violation_details(violations_channel) def pull_violation_details(xray_violation_detail_url) begin - detailResp = get_xray_violations_detail(xray_violation_detail_url) + detailResp_json = data_normalization(get_violations_detail(xray_violation_detail_url)) time = Fluent::Engine.now - detailResp_json = data_normalization(detailResp) @router.emit(@tag, time, detailResp_json) rescue => e puts "error: #{e}" @@ -63,7 +62,7 @@ def pull_violation_details(xray_violation_detail_url) end end - def get_xray_violations_detail(xray_violation_detail_url) + def get_violations_detail(xray_violation_detail_url) response = RestClient::Request.new( :method => :get, :url => xray_violation_detail_url, @@ -72,7 +71,7 @@ def get_xray_violations_detail(xray_violation_detail_url) ).execute do |response, request, result| case response.code when 200 - return response.to_str + return JSON.parse(response.to_s) else puts "error: #{response.to_json}" raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." @@ -80,8 +79,7 @@ def get_xray_violations_detail(xray_violation_detail_url) end end - def data_normalization(detailResp) - detailResp_json = JSON.parse(detailResp) + def data_normalization(detailResp_json) cve = [] cvss_v2_list = [] cvss_v3_list = [] @@ -130,8 +128,7 @@ def data_normalization(detailResp) detailResp_json['rules'] = rule_list end - impacted_artifacts = detailResp_json['impacted_artifacts'] - for impacted_artifact in impacted_artifacts do + detailResp_json['impacted_artifacts'].each do |impacted_artifact| matchdata = impacted_artifact.match /default\/(?[^\/]*)\/(?.*)/ impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " " impacted_artifact_url_list.append(impacted_artifact_url) @@ -141,7 +138,7 @@ def data_normalization(detailResp) end private - def get_xray_violations(xray_json) + def get_violations(xray_json) response = RestClient::Request.new( :method => :post, :url => @jpd_url + "/xray/api/v1/violations", @@ -152,7 +149,7 @@ def get_xray_violations(xray_json) ).execute do |response, request, result| case response.code when 200 - return response.to_str + return JSON.parse(response.to_str) else puts "error: #{response.to_json}" raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations. #{response.to_json}" diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index e3b669d..c5e7112 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -9,17 +9,27 @@ require 'xray' require 'date' require 'rspec' +require 'rest-client' RSpec.describe Xray do describe "#violation_details" do - let(:violation1){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } - let(:violation2){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch2", "issue_id": "55443"} } + let(:violation1){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), + "watch_name": "watch1", + "issue_id": "55444", + "violation_details_url": "http://www.com"} + } + let(:violation2){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), + "watch_name": "watch2", + "issue_id": "55443", + "violation_details_url": "http://www.com"} + } + + let(:violations) { Concurrent::Array.new } it "creates a future for every violation" do xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router) - violations = Concurrent::Array.new (1..5).each do |i| violations << i @@ -31,26 +41,30 @@ xray.violation_details(violations) end - xit "updates pos file for every violation" do - pos_file = double('pos_file') - - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @router) - violations = Concurrent::Array.new + it "updates pos file for every violation (cannot do exactly tests since stubs with Concurrent ruby are broken)" do + router = double('router') + pos_file_path = `pwd` + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, pos_file_path, router) - violations << violation1 - violations << violation2 + violations << JSON.parse(violation1.to_json) + violations << JSON.parse(violation2.to_json) - datetime = double (DateTime) - expect(datetime).to receive(:parse) promises = class_double("Concurrent::Promises").as_stubbed_const(:transfer_nested_constants => true) - allow(promises).to receive(:future) { |&block| block.call } + allow(promises).to receive(:future).and_yield(violation1).and_yield(violation2) + + rest_client = double("RestClient::Request") + allow(RestClient::Request).to receive(:new).and_return rest_client + allow(rest_client).to receive(:execute).and_return(JSON.parse({'impacted_artifacts': []}.to_json)) + + pos_file = double(PositionFile) + allow(PositionFile).to receive(:new).and_return pos_file + allow(pos_file).to receive(:write) fluent = class_double("Fluent::Engine").as_stubbed_const(:transfer_nested_constants => true) - expect(fluent).to receive(:now).and_return(DateTime.now) - pos_file = double (PositionFile) - expect(pos_file).to receive(:write).exactly(2).times + allow(fluent).to receive(:now).and_return(DateTime.now) + allow(router).to receive(:emit) - xray.violation_details(violations) + xray.violation_details(JSON.parse(violations.to_json)) end end From 97623418bc24ff61bcd10a80fa53e64569e49477 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Sun, 25 Jul 2021 22:05:53 -0700 Subject: [PATCH 36/44] Extracts methods to enable testing of processed logic - simplifies code by shortening lines in timer task --- .../lib/fluent/plugin/xray.rb | 25 ++++--- .../spec/xray_spec.rb | 70 ++++++++++++++++--- 2 files changed, 73 insertions(+), 22 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index ae4c1b9..4276f2b 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -20,20 +20,12 @@ def violations(date_since) timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } resp = get_violations(xray_json) - total_violation_count = resp['total_violations'] page_violation_count = resp['violations'].length puts "Total violations count is #{total_violation_count}" - if total_violation_count > 0 + if resp['total_violations'] > 0 puts "Number of Violations in page #{page_number} are #{page_violation_count}" - resp['violations'].each do |violation| - pos_file = PositionFile.new(@pos_file_path) - unless pos_file.processed?(violation) - violations_channel << violation - end - end - if page_violation_count == @batch_size - page_number += 1 - end + resp['violations'].each {|v| violations_channel = process(v, violations_channel) } + page_number += 1 if next_page?(page_violation_count) end end timer_task.execute @@ -137,6 +129,12 @@ def data_normalization(detailResp_json) return detailResp_json end + def process(violation, violations_channel) + pos_file = PositionFile.new(@pos_file_path) + violations_channel << violation unless pos_file.processed?(violation) + violations_channel + end + private def get_violations(xray_json) response = RestClient::Request.new( @@ -156,5 +154,10 @@ def get_violations(xray_json) end end end + + def next_page?(count) + count == @batch_size + end + end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index c5e7112..3857c72 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -15,26 +15,26 @@ RSpec.describe Xray do describe "#violation_details" do - let(:violation1){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), - "watch_name": "watch1", - "issue_id": "55444", - "violation_details_url": "http://www.com"} + let(:violation1){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), + "watch_name": "watch1", + "issue_id": "55444", + "violation_details_url": "http://www.com"} } - let(:violation2){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), - "watch_name": "watch2", - "issue_id": "55443", - "violation_details_url": "http://www.com"} + let(:violation2){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), + "watch_name": "watch2", + "issue_id": "55443", + "violation_details_url": "http://www.com"} } let(:violations) { Concurrent::Array.new } it "creates a future for every violation" do xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router) - + (1..5).each do |i| violations << i end - + promises = class_double("Concurrent::Promises").as_stubbed_const(:transfer_nested_constants => true) expect(promises).to receive(:future).exactly(5).times @@ -45,7 +45,7 @@ router = double('router') pos_file_path = `pwd` xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, pos_file_path, router) - + violations << JSON.parse(violation1.to_json) violations << JSON.parse(violation2.to_json) @@ -68,5 +68,53 @@ end end + describe "#violations" do + + end + + describe "#process" do + let(:violation1){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), + "watch_name": "watch1", + "issue_id": "55444", + "violation_details_url": "http://www.com"} + } + let(:violation2){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), + "watch_name": "watch2", + "issue_id": "55443", + "violation_details_url": "http://www.com"} + } + + let(:violations_channel) { Concurrent::Array.new } + + it "skips processed violation" do + pos_file_path = `pwd` + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router) + + violations_channel << violation1 + + pos_file = double(PositionFile) + allow(PositionFile).to receive(:new).and_return pos_file + allow(pos_file).to receive(:processed?).and_return true + + xray.process(violation1, violations_channel) + + expect(violations_channel.size).to eq 1 + end + + it "adds unprocessed violation to the channel" do + pos_file_path = `pwd` + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router) + + violations_channel << violation1 + + pos_file = double(PositionFile) + allow(PositionFile).to receive(:new).and_return pos_file + allow(pos_file).to receive(:processed?).and_return false + + xray.process(violation2, violations_channel) + + expect(violations_channel.size).to eq 2 + end + end end \ No newline at end of file From 1b79936a8ae56e19f9ee2c2cbb6e474db254b011 Mon Sep 17 00:00:00 2001 From: Sudhindra Rao Date: Mon, 26 Jul 2021 10:59:03 -0700 Subject: [PATCH 37/44] Verifies that the timer task is called --- .../fluent-plugin-jfrog-siem/spec/xray_spec.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index 3857c72..aad2711 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -69,6 +69,21 @@ end describe "#violations" do + it "runs a timer task to process violations periodically" do + batch_size = 25 + pos_file_path = `pwd` + router = double('router') + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, batch_size, pos_file_path, router) + + channel = double(Concurrent::Channel) + expect(Concurrent::Channel).to receive(:new).with(capacity: batch_size).and_return(channel) + + timer_task = double(Concurrent::TimerTask) + expect(Concurrent::TimerTask).to receive(:new).and_return timer_task + + expect(timer_task).to receive(:execute) + xray.violations(Date.today) + end end From 4f573bdefd1c3929efffb9d962703c35809d6f43 Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Tue, 27 Jul 2021 09:51:22 -0700 Subject: [PATCH 38/44] Sending tag param to xray --- .../lib/fluent/plugin/in_jfrog_siem.rb | 2 +- .../input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index 1ede215..efe613a 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -94,7 +94,7 @@ def run end date_since = last_created_date puts "Getting queries from #{date_since}" - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, router) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, router, @tag) violations_channel = xray.violations(date_since) xray.violation_details(violations_channel) sleep 100 diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 4276f2b..024d1e4 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -4,7 +4,7 @@ require "fluent/plugin/position_file" class Xray - def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file_path, router) + def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file_path, router, tag) @jpd_url = jpd_url @username = username @api_key = api_key @@ -12,6 +12,7 @@ def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file_p @batch_size = batch_size @pos_file_path = pos_file_path @router = router + @tag = tag end def violations(date_since) From 4ce2cffae8eda13938d9ac15bda45e65e801cf2e Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Tue, 27 Jul 2021 09:53:56 -0700 Subject: [PATCH 39/44] Adding spec changes for tag --- .../input/fluent-plugin-jfrog-siem/spec/xray_spec.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb index aad2711..284a9c0 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -29,7 +29,7 @@ let(:violations) { Concurrent::Array.new } it "creates a future for every violation" do - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router, @tag) (1..5).each do |i| violations << i @@ -44,7 +44,7 @@ it "updates pos file for every violation (cannot do exactly tests since stubs with Concurrent ruby are broken)" do router = double('router') pos_file_path = `pwd` - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, pos_file_path, router) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, pos_file_path, router, @tag) violations << JSON.parse(violation1.to_json) violations << JSON.parse(violation2.to_json) @@ -73,7 +73,7 @@ batch_size = 25 pos_file_path = `pwd` router = double('router') - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, batch_size, pos_file_path, router) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, batch_size, pos_file_path, router, @tag) channel = double(Concurrent::Channel) expect(Concurrent::Channel).to receive(:new).with(capacity: batch_size).and_return(channel) @@ -103,7 +103,7 @@ it "skips processed violation" do pos_file_path = `pwd` - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router, @tag) violations_channel << violation1 @@ -118,7 +118,7 @@ it "adds unprocessed violation to the channel" do pos_file_path = `pwd` - xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router) + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router, @tag) violations_channel << violation1 From 4f90bfb38f0da009945edb1efb16c1aa8ade1d39 Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Thu, 29 Jul 2021 11:49:24 -0700 Subject: [PATCH 40/44] Printing total violations var --- .../input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 024d1e4..124e91b 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -22,7 +22,7 @@ def violations(date_since) xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } resp = get_violations(xray_json) page_violation_count = resp['violations'].length - puts "Total violations count is #{total_violation_count}" + puts "Total violations count is #{resp['violations']}" if resp['total_violations'] > 0 puts "Number of Violations in page #{page_number} are #{page_violation_count}" resp['violations'].each {|v| violations_channel = process(v, violations_channel) } From f2a3373a256771ff8119ad9b04959698b7109381 Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Thu, 29 Jul 2021 12:00:00 -0700 Subject: [PATCH 41/44] total violations for print statement --- .../input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb index 124e91b..caa135c 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -22,7 +22,7 @@ def violations(date_since) xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } resp = get_violations(xray_json) page_violation_count = resp['violations'].length - puts "Total violations count is #{resp['violations']}" + puts "Total violations count is #{resp['total_violations']}" if resp['total_violations'] > 0 puts "Number of Violations in page #{page_number} are #{page_violation_count}" resp['violations'].each {|v| violations_channel = process(v, violations_channel) } From 1ff290ea3d218cecf29064dc4d3243396d905d1c Mon Sep 17 00:00:00 2001 From: Harrison Mitgang <9940778+hmitgang@users.noreply.github.com> Date: Fri, 30 Jul 2021 15:51:23 -0700 Subject: [PATCH 42/44] Create CI pipeline (#85) * init pipeline commit * cd source repo * Use concurrent_ruby branch --- pipelines.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 pipelines.yml diff --git a/pipelines.yml b/pipelines.yml new file mode 100644 index 0000000..54e7108 --- /dev/null +++ b/pipelines.yml @@ -0,0 +1,32 @@ +resources: + - name: source_repo + type: GitRepo + configuration: + gitProvider: github_integration + path: jfrog/log-analytics + branches: + include: concurrent_ruby + + +pipelines: + - name: siem_plugin_test + steps: + - name: siem_plugin_test_step + type: Bash + configuration: + inputResources: + - name: source_repo + trigger: true + runtime: + type: image + image: + custom: + name: ruby + tag: '3.0.2' + autoPull: true + execution: + onExecute: + - cd $res_source_repo_resourcePath/fluentd/plugins/input/fluent-plugin-jfrog-siem + - bundle --version + - bundle install + - rspec From 5536458a63f342c09fbb3cd17ee63cf235b501eb Mon Sep 17 00:00:00 2001 From: Harrison Mitgang Date: Fri, 30 Jul 2021 16:58:36 -0700 Subject: [PATCH 43/44] moved pipelines to ci directory --- pipelines.yml => fluentd/plugins/ci/pipelines.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pipelines.yml => fluentd/plugins/ci/pipelines.yml (100%) diff --git a/pipelines.yml b/fluentd/plugins/ci/pipelines.yml similarity index 100% rename from pipelines.yml rename to fluentd/plugins/ci/pipelines.yml From 74a1a6e08380467b1def26c8726288e0c9f7cbf0 Mon Sep 17 00:00:00 2001 From: MahithaB <60710901+MahithaB@users.noreply.github.com> Date: Mon, 2 Aug 2021 13:18:46 -0700 Subject: [PATCH 44/44] Adding test file --- .../test/plugin/test_in_jfrog_siem.rb | 37 +++++-------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb index 27227ee..a635395 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb @@ -11,35 +11,16 @@ class JfrogSiemInputTest < Test::Unit::TestCase end # Default configuration for tests - # CONFIG = %[ - # tag "partnership.test_tag" - # jpd_url "https://partnership.jfrog.io/" - # username "sudhindrar" - # apikey "AKCp8ihpNg2JE5PV3nRXZQsmMGmzX9VTX6wN51hQBFRC1CXQWzGrKQvFL1tsw7aochjoQXAZq" - # pos_file "test_pos.txt" - # wait_interval 30 - # ] - CONFIG = %[ - tag "sudhindra-xray-rt.test_tag" - jpd_url "https://sudhindra-xray-rt.jfrog.tech/" - username "admin" - apikey "AKCp8jQd1zP4oKv43SNgewrNwikd1iAQznfhSfx3T249eVMkGnJnSjCpNsuv8vtHWChKLfJ1w" - wait_interval 1 - from_date "2021-06-15" - pos_file_path "/Users/sudhindrar/ptreng/log-analytics/fluentd/plugins/input/fluent-plugin-jfrog-siem/" - ] - - # CONFIG = %[ - # tag "jfrog.xray.siem.vulnerabilities" - # jpd_url "JPDURL" - # username "admin" - # apikey "APIKEY" - # pos_file_path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/" - # wait_interval 10 - # from_date "2016-01-01" - # batch_size 25 - # ] + tag "jfrog.xray.siem.vulnerabilities" + jpd_url "JPDURL" + username "admin" + apikey "APIKEY" + pos_file_path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/" + wait_interval 10 + from_date "2016-01-01" + batch_size 25 + ] private