diff --git a/.gitignore b/.gitignore index 9f6676b..0f98557 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ .tox ./idea .idea/ -.DS_Store \ No newline at end of file +.DS_Store +**/test_pos.txt \ No newline at end of file diff --git a/fluentd/plugins/ci/pipelines.yml b/fluentd/plugins/ci/pipelines.yml new file mode 100644 index 0000000..54e7108 --- /dev/null +++ b/fluentd/plugins/ci/pipelines.yml @@ -0,0 +1,32 @@ +resources: + - name: source_repo + type: GitRepo + configuration: + gitProvider: github_integration + path: jfrog/log-analytics + branches: + include: concurrent_ruby + + +pipelines: + - name: siem_plugin_test + steps: + - name: siem_plugin_test_step + type: Bash + configuration: + inputResources: + - name: source_repo + trigger: true + runtime: + type: image + image: + custom: + name: ruby + tag: '3.0.2' + autoPull: true + execution: + onExecute: + - cd $res_source_repo_resourcePath/fluentd/plugins/input/fluent-plugin-jfrog-siem + - bundle --version + - bundle install + - rspec diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/.rspec b/fluentd/plugins/input/fluent-plugin-jfrog-siem/.rspec new file mode 100644 index 0000000..c99d2e7 --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/.rspec @@ -0,0 +1 @@ +--require spec_helper diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock new file mode 100644 index 0000000..b872189 --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Gemfile.lock @@ -0,0 +1,90 @@ +PATH + remote: . + specs: + fluent-plugin-jfrog-siem (1.0.0) + concurrent-ruby (~> 1.1.8) + concurrent-ruby-edge + fluentd (>= 0.14.10, < 2) + +GEM + remote: https://rubygems.org/ + specs: + concurrent-ruby (1.1.8) + concurrent-ruby-edge (0.6.0) + concurrent-ruby (~> 1.1.6) + cool.io (1.7.1) + diff-lcs (1.4.4) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) + fluentd (1.13.1) + bundler + cool.io (>= 1.4.5, < 2.0.0) + http_parser.rb (>= 0.5.1, < 0.7.0) + msgpack (>= 1.3.1, < 2.0.0) + serverengine (>= 2.2.2, < 3.0.0) + sigdump (~> 0.2.2) + strptime (>= 0.2.2, < 1.0.0) + tzinfo (>= 1.0, < 3.0) + tzinfo-data (~> 1.0) + webrick (>= 1.4.2, < 1.8.0) + yajl-ruby (~> 1.0) + http-accept (1.7.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + http_parser.rb (0.6.0) + mime-types (3.3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2021.0225) + msgpack (1.4.2) + netrc (0.11.0) + power_assert (2.0.0) + rake (12.3.3) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rspec (3.10.0) + rspec-core (~> 3.10.0) + rspec-expectations (~> 3.10.0) + rspec-mocks (~> 3.10.0) + rspec-core (3.10.1) + rspec-support (~> 3.10.0) + rspec-expectations (3.10.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.10.0) + rspec-mocks (3.10.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.10.0) + rspec-support (3.10.2) + serverengine (2.2.4) + sigdump (~> 0.2.2) + sigdump (0.2.4) + strptime (0.2.5) + test-unit (3.4.2) + power_assert + tzinfo (2.0.4) + concurrent-ruby (~> 1.0) + tzinfo-data (1.2021.1) + tzinfo (>= 1.0.0) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.7) + webrick (1.7.0) + yajl-ruby (1.4.1) + +PLATFORMS + x86_64-darwin-20 + +DEPENDENCIES + bundler (~> 2.0) + concurrent-ruby (~> 1.1.8) + concurrent-ruby-edge + fluent-plugin-jfrog-siem! + rake (~> 12.0) + rest-client (~> 2.0) + rspec (~> 3.10.0) + test-unit (~> 3.0) + +BUNDLED WITH + 2.2.3 diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/Rakefile b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Rakefile index 9caf936..a388768 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/Rakefile +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/Rakefile @@ -4,7 +4,7 @@ Bundler::GemHelper.install_tasks require "rake/testtask" Rake::TestTask.new(:test) do |t| - t.libs.push("lib", "test") + t.libs.push("lib", "test", 'lib/fluent/plugin') t.test_files = FileList["test/**/test_*.rb"] t.verbose = true t.warning = false diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec b/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec index e353840..7217dbc 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/fluent-plugin-jfrog-siem.gemspec @@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| spec.name = "fluent-plugin-jfrog-siem" - spec.version = "1.0.0" + spec.version = "2.0.0" spec.authors = ["John Peterson", "Mahitha Byreddy"] spec.email = ["johnp@jfrog.com", "mahithab@jfrog.com"] @@ -24,7 +24,12 @@ Gem::Specification.new do |spec| spec.add_development_dependency "rake", "~> 12.0" spec.add_development_dependency "test-unit", "~> 3.0" spec.add_development_dependency "rest-client", "~> 2.0" - spec.add_development_dependency "thread", "~> 0.2.2" - spec.add_runtime_dependency "thread", "~> 0.2.2" + spec.add_development_dependency "concurrent-ruby", "~> 1.1.8" + spec.add_development_dependency "concurrent-ruby-edge", '>= 0' + spec.add_development_dependency 'rspec', '~> 3.10.0' + + spec.add_runtime_dependency "rest-client", "~> 2.0" + spec.add_runtime_dependency "concurrent-ruby", "~> 1.1.8" + spec.add_runtime_dependency "concurrent-ruby-edge", '>= 0' spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"] end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb index d04d13a..efe613a 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/in_jfrog_siem.rb @@ -14,10 +14,10 @@ # limitations under the License. require "fluent/plugin/input" require "rest-client" -require "thread/pool" -require "json" require "date" require "uri" +require "fluent/plugin/xray" +require "fluent/plugin/position_file" module Fluent module Plugin @@ -32,11 +32,10 @@ class JfrogSiemInput < Fluent::Plugin::Input config_param :jpd_url, :string, default: "" config_param :username, :string, default: "" config_param :apikey, :string, default: "" - config_param :pos_file, :string, default: "" config_param :batch_size, :integer, default: 25 - config_param :thread_count, :integer, default: 5 config_param :wait_interval, :integer, default: 60 - + config_param :from_date, :string, default: "" + config_param :pos_file_path, :string, default: "" # `configure` is called before `start`. # 'conf' is a `Hash` that includes the configuration parameters. @@ -59,22 +58,14 @@ def configure(conf) raise Fluent::ConfigError, "Must define the API Key to use for authentication." end - if @pos_file == "" - raise Fluent::ConfigError, "Must define a position file to record last SIEM violation pulled." - end - - if @thread_count < 1 - raise Fluent::ConfigError, "Must define at least one thread to process violation details." - end - - if @thread_count > @batch_size - raise Fluent::ConfigError, "Violation detail url thread count exceeds batch size." - end - if @wait_interval < 1 raise Fluent::ConfigError, "Wait interval must be greater than 1 to wait between pulling new events." end + if @from_date == "" + puts "From date not specified, so getting violations from current date if pos_file doesn't exist" + end + end @@ -94,108 +85,19 @@ def shutdown def run - call_home(@jpd_url) - # runs the violation pull - last_created_date_string = get_last_item_create_date() - begin - last_created_date = DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ") - rescue - last_created_date = DateTime.parse("1970-01-01T00:00:00Z").strftime("%Y-%m-%dT%H:%M:%SZ") - end - offset_count=1 - left_violations=0 - waiting_for_violations = false - xray_json={"filters": { "created_from": last_created_date }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": offset_count } } - - while true - # Grab the batch of records - resp=get_xray_violations(xray_json, @jpd_url) - number_of_violations = JSON.parse(resp)['total_violations'] - if left_violations <= 0 - left_violations = number_of_violations - end - - xray_violation_urls_list = [] - for index in 0..JSON.parse(resp)['violations'].length-1 do - # Get the violation - item = JSON.parse(resp)['violations'][index] - - # Get the created date and check if we should skip (already processed) or process this record. - created_date_string = item['created'] - created_date = DateTime.parse(created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ") + # call_home(@jpd_url) - # Determine if we need to persist this record or not - persistItem = true - if waiting_for_violations - if created_date <= last_created_date - # "not persisting it - waiting for violations" - # waiting and same last timestamp (left violations in batch) - persistItem = false - end - if created_date > last_created_date - # new violation while waiting - persistItem = true - waiting_for_violations = false - end - else - if created_date < last_created_date - # "persisting everything" - persistItem = true - end - end + last_created_date = get_last_item_create_date() - # Publish the record to fluentd - if persistItem - - now = Fluent::Engine.now - router.emit(@tag, now, item) - - # write to the pos_file created_date_string - open(@pos_file, 'a') do |f| - f << "#{created_date_string}\n" - end - - # Mark this as the last record successfully processed - last_created_date_string = created_date_string - last_created_date = created_date - - # Grab violation detail url and add to url list to process w/ thread pool - xray_violation_details_url=item['violation_details_url'] - xray_violation_urls_list.append(xray_violation_details_url) - end - end - - # iterate over url array adding to thread pool each url. - # limit max workers to thread count to prevent overloading xray. - thread_pool = Thread.pool(thread_count) - thread_pool.process { - for xray_violation_url in xray_violation_urls_list do - pull_violation_details(xray_violation_url) - end - } - thread_pool.shutdown - - # reduce left violations by jump size (not all batches have full item count??) - left_violations = left_violations - @batch_size - if left_violations <= 0 - waiting_for_violations = true - sleep(@wait_interval) - else - # Grab the next record to process for the violation details url - waiting_for_violations = false - offset_count = offset_count + 1 - xray_json={"filters": { "created_from": last_created_date_string }, "pagination": {"order_by": "created","limit": @batch_size , "offset": offset_count } } - end - end - end - - - # pull the last item create date from the pos_file return created_date_string - def get_last_item_create_date() - if(!(File.exist?(@pos_file))) - @pos_file = File.new(@pos_file, "w") + if (@from_date != "") + last_created_date = DateTime.parse(@from_date).strftime("%Y-%m-%dT%H:%M:%SZ") end - return IO.readlines(@pos_file).last + date_since = last_created_date + puts "Getting queries from #{date_since}" + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, router, @tag) + violations_channel = xray.violations(date_since) + xray.violation_details(violations_channel) + sleep 100 end #call home functionality @@ -209,119 +111,27 @@ def call_home(jpd_url) :password => @apikey, :headers => { :accept => :json, :content_type => :json} ).execute do |response, request, result| - puts "Posting call home information" + puts "Posting call home information" end end - # queries the xray API for violations based upon the input json - def get_xray_violations_detail(xray_violation_detail_url) - response = RestClient::Request.new( - :method => :get, - :url => xray_violation_detail_url, - :user => @username, - :password => @apikey - ).execute do |response, request, result| - case response.code - when 200 - return response.to_str - else - raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." - end - end - end - - - # queries the xray API for violations based upon the input json - def get_xray_violations(xray_json, jpd_url) - response = RestClient::Request.new( - :method => :post, - :url => jpd_url + "/xray/api/v1/violations", - :payload => xray_json.to_json, - :user => @username, - :password => @apikey, - :headers => { :accept => :json, :content_type => :json} - ).execute do |response, request, result| - case response.code - when 200 - return response.to_str - else - raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." - end - end - end - - # normalizes Xray data according to common information models for all log-vendors - def data_normalization(detailResp) - detailResp_json = JSON.parse(detailResp) - cve = [] - cvss_v2_list = [] - cvss_v3_list = [] - policy_list = [] - rule_list = [] - impacted_artifact_url_list = [] - if detailResp_json.key?('properties') - properties = detailResp_json['properties'] - for index in 0..properties.length-1 do - if properties[index].key?('cve') - cve.push(properties[index]['cve']) - end - if properties[index].key?('cvss_v2') - cvss_v2_list.push(properties[index]['cvss_v2']) - end - if properties[index].key?('cvss_v3') - cvss_v3_list.push(properties[index]['cvss_v3']) - end - end - - detailResp_json["cve"] = cve.sort.reverse[0] - cvss_v2 = cvss_v2_list.sort.reverse[0] - cvss_v3 = cvss_v3_list.sort.reverse[0] - if !cvss_v3.nil? - cvss = cvss_v3 - elsif !cvss_v2.nil? - cvss = cvss_v2 - end - cvss_score = cvss[0..2] - cvss_version = cvss.split(':')[1][0..2] - detailResp_json["cvss_score"] = cvss_score - detailResp_json["cvss_version"] = cvss_version - end - - if detailResp_json.key?('matched_policies') - matched_policies = detailResp_json['matched_policies'] - for index in 0..matched_policies.length-1 do - if matched_policies[index].key?('policy') - policy_list.push(matched_policies[index]['policy']) - end - if matched_policies[index].key?('rule') - rule_list.push(matched_policies[index]['rule']) - end - end - detailResp_json['policies'] = policy_list - detailResp_json['rules'] = rule_list - end - - impacted_artifacts = detailResp_json['impacted_artifacts'] - for impacted_artifact in impacted_artifacts do - matchdata = impacted_artifact.match /default\/(?[^\/]*)\/(?.*)/ - impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " " - impacted_artifact_url_list.append(impacted_artifact_url) + # pull the last item create date from the pos_file return created_date_string + def get_last_item_create_date() + recent_pos_file = get_recent_pos_file() + if recent_pos_file != nil + last_created_date_string = IO.readlines(recent_pos_file).last + return DateTime.parse(last_created_date_string).strftime("%Y-%m-%dT%H:%M:%SZ") + else + return DateTime.now.strftime("%Y-%m-%dT%H:%M:%SZ") end - detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list - return detailResp_json end - def pull_violation_details(xray_violation_detail_url) - begin - detailResp=get_xray_violations_detail(xray_violation_detail_url) - time = Fluent::Engine.now - detailResp_json = data_normalization(detailResp) - router.emit(@tag, time, detailResp_json) - rescue - raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}" - end + def get_recent_pos_file() + pos_file = @pos_file_path + "*.siem.pos" + return Dir.glob(pos_file).sort.last end end end end + diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb new file mode 100644 index 0000000..1c7ac99 --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/position_file.rb @@ -0,0 +1,32 @@ +class PositionFile + + def initialize(pos_file_path) + @pos_file_path = pos_file_path + end + + def processed?(violation) + File.exist?(pos_file_name(violation)) && found?(violation) + end + + def write(violation) + File.open(pos_file_name(violation), 'a') do |f| + f << violation_entry(violation) + f << "\n" + end + end + + private + def found?(violation) + return File.open(pos_file_name(violation)) { |f| f.find { |line| line.include? violation_entry(violation) } } + end + + def violation_entry(violation) + created_date = DateTime.parse(violation['created']).strftime("%Y-%m-%dT%H:%M:%SZ") + [created_date, violation['watch_name'], violation['issue_id']].join(',') + end + + def pos_file_name(violation) + pos_file_date = DateTime.parse(violation['created']).strftime("%Y-%m-%d") + @pos_file_path + "jfrog_siem_log_#{pos_file_date}.siem.pos" + end +end \ No newline at end of file diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/violations.json b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/violations.json new file mode 100644 index 0000000..0585695 --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/violations.json @@ -0,0 +1,380 @@ +{ + "total_violations":54, + "violations":[ + { + "description":"Apache Software Foundation (ASF) Struts before 1.2.9 allows remote attackers to bypass validation via a request with a 'org.apache.struts.taglib.html.Constants.CANCEL' parameter, which causes the action to be canceled but would not be detected from applications that do not use the isCancelled check.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.1" + ], + "created":"2021-06-16T21:22:18Z", + "watch_name":"maven-watch1", + "issue_id":"XRAY-55418", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=ff9cf61f42a095cd97ea0ec0&issue_id=XRAY-55418&comp_id=gav:%2F%2Fstruts:struts:1.1", + "impacted_artifacts":[ + "default/maven-repo-1/struts/struts/1.1/struts-1.1.jar" + ] + }, + { + "description":"The Apache Software License, Version 1.1", + "severity":"High", + "type":"License", + "infected_components":[ + "gav://struts:struts:1.1" + ], + "created":"2021-06-16T21:22:18Z", + "watch_name":"license-watch", + "issue_id":"Apache-1.1", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=b241c1986818d68993093e35&issue_id=Apache-1.1&comp_id=gav:%2F%2Fstruts:struts:1.1", + "impacted_artifacts":[ + "default/maven-repo-1/struts/struts/1.1/struts-1.1.jar" + ] + }, + { + "description":"The MultiPageValidator implementation in Apache Struts 1 1.1 through 1.3.10 allows remote attackers to bypass intended access restrictions via a modified page parameter.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.1" + ], + "created":"2021-06-16T21:22:18Z", + "watch_name":"maven-watch1", + "issue_id":"XRAY-55648", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=ff9cf61f42a095cd97ea0ec0&issue_id=XRAY-55648&comp_id=gav:%2F%2Fstruts:struts:1.1", + "impacted_artifacts":[ + "default/maven-repo-1/struts/struts/1.1/struts-1.1.jar" + ] + }, + { + "description":"Cross-site scripting (XSS) vulnerability in Apache Struts before 1.2.9-162.31.1 on SUSE Linux Enterprise (SLE) 11, before 1.2.9-108.2 on SUSE openSUSE 10.3, before 1.2.9-198.2 on SUSE openSUSE 11.0, and before 1.2.9-162.163.2 on SUSE openSUSE 11.1 allows remote attackers to inject arbitrary web script or HTML via unspecified vectors related to \"insufficient quoting of parameters.\"", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.1" + ], + "created":"2021-06-16T21:22:18Z", + "watch_name":"maven-watch1", + "issue_id":"XRAY-55444", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=ff9cf61f42a095cd97ea0ec0&issue_id=XRAY-55444&comp_id=gav:%2F%2Fstruts:struts:1.1", + "impacted_artifacts":[ + "default/maven-repo-1/struts/struts/1.1/struts-1.1.jar" + ] + }, + { + "description":"Cross-site scripting (XSS) vulnerability in (1) LookupDispatchAction and possibly (2) DispatchAction and (3) ActionDispatcher in Apache Software Foundation (ASF) Struts before 1.2.9 allows remote attackers to inject arbitrary web script or HTML via the parameter name, which is not filtered in the resulting error message.", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.1" + ], + "created":"2021-06-16T21:22:18Z", + "watch_name":"maven-watch1", + "issue_id":"XRAY-55420", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=ff9cf61f42a095cd97ea0ec0&issue_id=XRAY-55420&comp_id=gav:%2F%2Fstruts:struts:1.1", + "impacted_artifacts":[ + "default/maven-repo-1/struts/struts/1.1/struts-1.1.jar" + ] + }, + { + "description":"ActionForm in Apache Software Foundation (ASF) Struts before 1.2.9 with BeanUtils 1.7 allows remote attackers to cause a denial of service via a multipart/form-data encoded form with a parameter name that references the public getMultipartRequestHandler method, which provides further access to elements in the CommonsMultipartRequestHandler implementation and BeanUtils.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.1" + ], + "created":"2021-06-16T21:22:18Z", + "watch_name":"maven-watch1", + "issue_id":"XRAY-55419", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=ff9cf61f42a095cd97ea0ec0&issue_id=XRAY-55419&comp_id=gav:%2F%2Fstruts:struts:1.1", + "impacted_artifacts":[ + "default/maven-repo-1/struts/struts/1.1/struts-1.1.jar" + ] + }, + { + "description":"The MultiPageValidator implementation in Apache Struts 1 1.1 through 1.3.10 allows remote attackers to bypass intended access restrictions via a modified page parameter.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.2.4" + ], + "created":"2021-06-16T21:22:27Z", + "watch_name":"maven-watch-2", + "issue_id":"XRAY-55648", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2a3ee020bf1a86b84d122c0c&issue_id=XRAY-55648&comp_id=gav:%2F%2Fstruts:struts:1.2.4", + "impacted_artifacts":[ + "default/maven-repo-2/struts/struts/1.2.4/struts-1.2.4.jar" + ] + }, + { + "description":"ActionForm in Apache Software Foundation (ASF) Struts before 1.2.9 with BeanUtils 1.7 allows remote attackers to cause a denial of service via a multipart/form-data encoded form with a parameter name that references the public getMultipartRequestHandler method, which provides further access to elements in the CommonsMultipartRequestHandler implementation and BeanUtils.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.2.4" + ], + "created":"2021-06-16T21:22:27Z", + "watch_name":"maven-watch-2", + "issue_id":"XRAY-55419", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2a3ee020bf1a86b84d122c0c&issue_id=XRAY-55419&comp_id=gav:%2F%2Fstruts:struts:1.2.4", + "impacted_artifacts":[ + "default/maven-repo-2/struts/struts/1.2.4/struts-1.2.4.jar" + ] + }, + { + "description":"Cross-site scripting (XSS) vulnerability in (1) LookupDispatchAction and possibly (2) DispatchAction and (3) ActionDispatcher in Apache Software Foundation (ASF) Struts before 1.2.9 allows remote attackers to inject arbitrary web script or HTML via the parameter name, which is not filtered in the resulting error message.", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.2.4" + ], + "created":"2021-06-16T21:22:27Z", + "watch_name":"maven-watch-2", + "issue_id":"XRAY-55420", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2a3ee020bf1a86b84d122c0c&issue_id=XRAY-55420&comp_id=gav:%2F%2Fstruts:struts:1.2.4", + "impacted_artifacts":[ + "default/maven-repo-2/struts/struts/1.2.4/struts-1.2.4.jar" + ] + }, + { + "description":"The Apache Software License, Version 2.0", + "severity":"High", + "type":"License", + "infected_components":[ + "gav://struts:struts:1.2.4" + ], + "created":"2021-06-16T21:22:27Z", + "watch_name":"license-watch", + "issue_id":"Apache-2.0", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=b241c1986818d68993093e35&issue_id=Apache-2.0&comp_id=gav:%2F%2Fstruts:struts:1.2.4", + "impacted_artifacts":[ + "default/maven-repo-2/struts/struts/1.2.4/struts-1.2.4.jar" + ] + }, + { + "description":"Cross-site scripting (XSS) vulnerability in Apache Struts before 1.2.9-162.31.1 on SUSE Linux Enterprise (SLE) 11, before 1.2.9-108.2 on SUSE openSUSE 10.3, before 1.2.9-198.2 on SUSE openSUSE 11.0, and before 1.2.9-162.163.2 on SUSE openSUSE 11.1 allows remote attackers to inject arbitrary web script or HTML via unspecified vectors related to \"insufficient quoting of parameters.\"", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.2.4" + ], + "created":"2021-06-16T21:22:27Z", + "watch_name":"maven-watch-2", + "issue_id":"XRAY-55444", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2a3ee020bf1a86b84d122c0c&issue_id=XRAY-55444&comp_id=gav:%2F%2Fstruts:struts:1.2.4", + "impacted_artifacts":[ + "default/maven-repo-2/struts/struts/1.2.4/struts-1.2.4.jar" + ] + }, + { + "description":"Apache Software Foundation (ASF) Struts before 1.2.9 allows remote attackers to bypass validation via a request with a 'org.apache.struts.taglib.html.Constants.CANCEL' parameter, which causes the action to be canceled but would not be detected from applications that do not use the isCancelled check.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://struts:struts:1.2.4" + ], + "created":"2021-06-16T21:22:27Z", + "watch_name":"maven-watch-2", + "issue_id":"XRAY-55418", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2a3ee020bf1a86b84d122c0c&issue_id=XRAY-55418&comp_id=gav:%2F%2Fstruts:struts:1.2.4", + "impacted_artifacts":[ + "default/maven-repo-2/struts/struts/1.2.4/struts-1.2.4.jar" + ] + }, + { + "description":"Unicode Terms of Use", + "severity":"High", + "type":"License", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:37Z", + "watch_name":"license-watch", + "issue_id":"Unicode-TOU", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=b241c1986818d68993093e35&issue_id=Unicode-TOU&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"BSD 3-Clause \"New\" or \"Revised\" License", + "severity":"High", + "type":"License", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:37Z", + "watch_name":"license-watch", + "issue_id":"BSD-3-Clause", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=b241c1986818d68993093e35&issue_id=BSD-3-Clause&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"The Apache Software License, Version 2.0", + "severity":"High", + "type":"License", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:37Z", + "watch_name":"license-watch", + "issue_id":"Apache-2.0", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=b241c1986818d68993093e35&issue_id=Apache-2.0&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"The OGNL extensive expression evaluation capability in XWork in Struts 2.0.0 through 2.1.8.1, as used in Atlassian Fisheye, Crucible, and possibly other products, uses a permissive whitelist, which allows remote attackers to modify server-side context objects and bypass the \"#\" protection mechanism in ParameterInterceptors via the (1) #context, (2) #_memberAccess, (3) #root, (4) #this, (5) #_typeResolver, (6) #_classResolver, (7) #_traceEvaluations, (8) #_lastEvaluation, (9) #_keepLastEvaluation, and possibly other OGNL context variables, a different vulnerability than CVE-2008-6504.", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:37Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-55471", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-55471&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Academic Free License v2.1", + "severity":"High", + "type":"License", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:37Z", + "watch_name":"license-watch", + "issue_id":"AFL-2.1", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=b241c1986818d68993093e35&issue_id=AFL-2.1&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"ParametersInterceptor in OpenSymphony XWork 2.0.x before 2.0.6 and 2.1.x before 2.1.2, as used in Apache Struts and other products, does not properly restrict # (pound sign) references to context objects, which allows remote attackers to execute Object-Graph Navigation Language (OGNL) statements and modify server-side context objects, as demonstrated by use of a \\u0023 representation for the # character.", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-55446", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-55446&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Multiple cross-site scripting (XSS) vulnerabilities in Apache Struts 2.0.x before 2.0.11.1 and 2.1.x before 2.1.1 allow remote attackers to inject arbitrary web script or HTML via vectors associated with improper handling of (1) \" (double quote) characters in the href attribute of an s:a tag and (2) parameters in the action attribute of an s:url tag.", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-55448", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-55448&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Forced OGNL evaluation, when evaluated on raw user input in tag attributes, may lead to remote code execution. Affected software : Apache Struts 2.0.0 - Struts 2.5.25.", + "severity":"Critical", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-129844", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-129844&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"The ExceptionDelegator component in Apache Struts before 2.2.3.1 interprets parameter values as OGNL expressions during certain exception handling for mismatched data types of properties, which allows remote attackers to execute arbitrary Java code via a crafted parameter.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-55522", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-55522&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Apache Struts 2 before 2.3.14.3 allows remote attackers to execute arbitrary OGNL code via a request with a crafted value that contains both \"${}\" and \"%{}\" sequences, which causes the OGNL code to be evaluated twice.", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-55575", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-55575&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Apache Struts JSP Page Handling Unspecified XSS", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-81164", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-81164&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Apache Struts Tag Handling XSS", + "severity":"Medium", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-81187", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-81187&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + }, + { + "description":"Apache Struts EL / OGNL Interpretation Unspecified Remote Code Execution", + "severity":"High", + "type":"Security", + "infected_components":[ + "gav://org.apache.struts:struts2-core:2.0.9" + ], + "created":"2021-06-16T21:22:38Z", + "watch_name":"maven-watch-3", + "issue_id":"XRAY-87424", + "violation_details_url":"http://localhost:8046/xray/api/v1/violations?watch_id=2c5ecab8172f24086f4b3bf4&issue_id=XRAY-87424&comp_id=gav:%2F%2Forg.apache.struts:struts2-core:2.0.9", + "impacted_artifacts":[ + "default/maven-repo-3/org/apache/struts/struts2-core/2.0.9/struts2-core-2.0.9.jar" + ] + } + ] +} \ No newline at end of file diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb new file mode 100644 index 0000000..caa135c --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/lib/fluent/plugin/xray.rb @@ -0,0 +1,164 @@ +require 'concurrent' +require 'concurrent-edge' +require 'json' +require "fluent/plugin/position_file" + +class Xray + def initialize(jpd_url, username, api_key, wait_interval, batch_size, pos_file_path, router, tag) + @jpd_url = jpd_url + @username = username + @api_key = api_key + @wait_interval = wait_interval + @batch_size = batch_size + @pos_file_path = pos_file_path + @router = router + @tag = tag + end + + def violations(date_since) + violations_channel = Concurrent::Channel.new(capacity: @batch_size) + page_number = 1 + timer_task = Concurrent::TimerTask.new(execution_interval: @wait_interval, timeout_interval: 30) do + xray_json = {"filters": { "created_from": date_since }, "pagination": {"order_by": "created","limit": @batch_size ,"offset": page_number } } + resp = get_violations(xray_json) + page_violation_count = resp['violations'].length + puts "Total violations count is #{resp['total_violations']}" + if resp['total_violations'] > 0 + puts "Number of Violations in page #{page_number} are #{page_violation_count}" + resp['violations'].each {|v| violations_channel = process(v, violations_channel) } + page_number += 1 if next_page?(page_violation_count) + end + end + timer_task.execute + + violations_channel + end + + def violation_details(violations_channel) + violations_channel.each do |v| + Concurrent::Promises.future(v) do |v| + pull_violation_details(v['violation_details_url']) + pos_file = PositionFile.new(@pos_file_path) + pos_file.write(v) + end + end + end + + def pull_violation_details(xray_violation_detail_url) + begin + detailResp_json = data_normalization(get_violations_detail(xray_violation_detail_url)) + time = Fluent::Engine.now + @router.emit(@tag, time, detailResp_json) + rescue => e + puts "error: #{e}" + raise Fluent::ConfigError, "Error pulling violation details url #{xray_violation_detail_url}: #{e}" + end + end + + def get_violations_detail(xray_violation_detail_url) + response = RestClient::Request.new( + :method => :get, + :url => xray_violation_detail_url, + :user => @username, + :password => @api_key + ).execute do |response, request, result| + case response.code + when 200 + return JSON.parse(response.to_s) + else + puts "error: #{response.to_json}" + raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations." + end + end + end + + def data_normalization(detailResp_json) + cve = [] + cvss_v2_list = [] + cvss_v3_list = [] + policy_list = [] + rule_list = [] + impacted_artifact_url_list = [] + if detailResp_json.key?('properties') + properties = detailResp_json['properties'] + for index in 0..properties.length-1 do + if properties[index].key?('cve') + cve.push(properties[index]['cve']) + end + if properties[index].key?('cvss_v2') + cvss_v2_list.push(properties[index]['cvss_v2']) + end + if properties[index].key?('cvss_v3') + cvss_v3_list.push(properties[index]['cvss_v3']) + end + end + + detailResp_json["cve"] = cve.sort.reverse[0] + cvss_v2 = cvss_v2_list.sort.reverse[0] + cvss_v3 = cvss_v3_list.sort.reverse[0] + if !cvss_v3.nil? + cvss = cvss_v3 + elsif !cvss_v2.nil? + cvss = cvss_v2 + end + cvss_score = cvss[0..2] + cvss_version = cvss.split(':')[1][0..2] + detailResp_json["cvss_score"] = cvss_score + detailResp_json["cvss_version"] = cvss_version + end + + if detailResp_json.key?('matched_policies') + matched_policies = detailResp_json['matched_policies'] + for index in 0..matched_policies.length-1 do + if matched_policies[index].key?('policy') + policy_list.push(matched_policies[index]['policy']) + end + if matched_policies[index].key?('rule') + rule_list.push(matched_policies[index]['rule']) + end + end + detailResp_json['policies'] = policy_list + detailResp_json['rules'] = rule_list + end + + detailResp_json['impacted_artifacts'].each do |impacted_artifact| + matchdata = impacted_artifact.match /default\/(?[^\/]*)\/(?.*)/ + impacted_artifact_url = matchdata['repo_name'] + ":" + matchdata['path'] + " " + impacted_artifact_url_list.append(impacted_artifact_url) + end + detailResp_json['impacted_artifacts_url'] = impacted_artifact_url_list + return detailResp_json + end + + def process(violation, violations_channel) + pos_file = PositionFile.new(@pos_file_path) + violations_channel << violation unless pos_file.processed?(violation) + violations_channel + end + + private + def get_violations(xray_json) + response = RestClient::Request.new( + :method => :post, + :url => @jpd_url + "/xray/api/v1/violations", + :payload => xray_json.to_json, + :user => @username, + :password => @api_key, + :headers => { :accept => :json, :content_type => :json } + ).execute do |response, request, result| + case response.code + when 200 + return JSON.parse(response.to_str) + else + puts "error: #{response.to_json}" + raise Fluent::ConfigError, "Cannot reach Artifactory URL to pull Xray SIEM violations. #{response.to_json}" + end + end + end + + def next_page?(count) + count == @batch_size + end + +end + diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/position_file_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/position_file_spec.rb new file mode 100644 index 0000000..d524b1f --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/position_file_spec.rb @@ -0,0 +1,56 @@ +[ + File.join(File.dirname(__FILE__), '..'), + File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'), + File.join(File.dirname(__FILE__), '..', 'spec'), +].each do |dir| + $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir) +end + +require 'position_file' +require 'date' +require 'rspec' + + +RSpec.describe PositionFile do + describe "#processed?" do + let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } + + pos_file_date = DateTime.parse(Date.today.to_s).strftime("%Y-%m-%d") + temp_pos_file = "jfrog_siem_log_#{pos_file_date}.pos" + + it "returns false when a violation has not been processed" do + pos_file = PositionFile.new(`pwd`) + allow(File).to receive(:open).and_yield [] + + expect(pos_file.processed?(JSON.parse(violation.to_json))).to be_falsey + end + + it "returns true when a violation was found in the pos file" do + pos_file = PositionFile.new(`pwd`) + + matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',') + another_violation = [violation[:created], "watch2", "12345"].join(',') + allow(File).to receive(:exist?).and_return true + allow(File).to receive(:open).and_yield [matching_violation, another_violation] + + expect(pos_file.processed?(JSON.parse(violation.to_json))).to be_truthy + end + + end + + describe "#write" do + let(:violation){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), "watch_name": "watch1", "issue_id": "55444"} } + + it "returns false when a violation has not been processed" do + pos_file = PositionFile.new(`pwd`) + + result = [] + allow(File).to receive(:open).and_yield result + + pos_file.write(JSON.parse(violation.to_json)) + + matching_violation = [violation[:created], violation[:watch_name], violation[:issue_id]].join(',') + expect(result.include? matching_violation).to be_truthy + end + end +end \ No newline at end of file diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/spec_helper.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/spec_helper.rb new file mode 100644 index 0000000..cb320f2 --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/spec_helper.rb @@ -0,0 +1,111 @@ +# This file was generated by the `rspec --init` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration + +[ + File.join(File.dirname(__FILE__), '..'), + File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'), + File.join(File.dirname(__FILE__), '..', 'spec'), +].each do |dir| + $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir) +end + +require 'xray' + +RSpec.configure do |config| + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + +# The settings below are suggested to provide a good initial experience +# with RSpec, but feel free to customize to your heart's content. +=begin + # This allows you to limit a spec run to individual examples or groups + # you care about by tagging them with `:focus` metadata. When nothing + # is tagged with `:focus`, all examples get run. RSpec also provides + # aliases for `it`, `describe`, and `context` that include `:focus` + # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + config.filter_run_when_matching :focus + + # Allows RSpec to persist some state between runs in order to support + # the `--only-failures` and `--next-failure` CLI options. We recommend + # you configure your source control system to ignore this file. + config.example_status_persistence_file_path = "spec/examples.txt" + + # Limits the available syntax to the non-monkey patched syntax that is + # recommended. For more details, see: + # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ + # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ + # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode + config.disable_monkey_patching! + + # This setting enables warnings. It's recommended, but in some cases may + # be too noisy due to issues in dependencies. + config.warnings = true + + # Many RSpec users commonly either run the entire suite or an individual + # file, and it's useful to allow more verbose output when running an + # individual spec file. + if config.files_to_run.one? + # Use the documentation formatter for detailed output, + # unless a formatter has already been configured + # (e.g. via a command-line flag). + config.default_formatter = "doc" + end + + # Print the 10 slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. + config.profile_examples = 10 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed +=end +end diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb new file mode 100644 index 0000000..284a9c0 --- /dev/null +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/spec/xray_spec.rb @@ -0,0 +1,135 @@ +[ + File.join(File.dirname(__FILE__), '..'), + File.join(File.dirname(__FILE__), '..', 'lib/fluent/plugin'), + File.join(File.dirname(__FILE__), '..', 'spec'), +].each do |dir| + $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir) +end + +require 'xray' +require 'date' +require 'rspec' +require 'rest-client' + + +RSpec.describe Xray do + describe "#violation_details" do + + let(:violation1){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), + "watch_name": "watch1", + "issue_id": "55444", + "violation_details_url": "http://www.com"} + } + let(:violation2){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), + "watch_name": "watch2", + "issue_id": "55443", + "violation_details_url": "http://www.com"} + } + + let(:violations) { Concurrent::Array.new } + + it "creates a future for every violation" do + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router, @tag) + + (1..5).each do |i| + violations << i + end + + promises = class_double("Concurrent::Promises").as_stubbed_const(:transfer_nested_constants => true) + expect(promises).to receive(:future).exactly(5).times + + xray.violation_details(violations) + end + + it "updates pos file for every violation (cannot do exactly tests since stubs with Concurrent ruby are broken)" do + router = double('router') + pos_file_path = `pwd` + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, pos_file_path, router, @tag) + + violations << JSON.parse(violation1.to_json) + violations << JSON.parse(violation2.to_json) + + promises = class_double("Concurrent::Promises").as_stubbed_const(:transfer_nested_constants => true) + allow(promises).to receive(:future).and_yield(violation1).and_yield(violation2) + + rest_client = double("RestClient::Request") + allow(RestClient::Request).to receive(:new).and_return rest_client + allow(rest_client).to receive(:execute).and_return(JSON.parse({'impacted_artifacts': []}.to_json)) + + pos_file = double(PositionFile) + allow(PositionFile).to receive(:new).and_return pos_file + allow(pos_file).to receive(:write) + + fluent = class_double("Fluent::Engine").as_stubbed_const(:transfer_nested_constants => true) + allow(fluent).to receive(:now).and_return(DateTime.now) + allow(router).to receive(:emit) + + xray.violation_details(JSON.parse(violations.to_json)) + end + end + + describe "#violations" do + it "runs a timer task to process violations periodically" do + batch_size = 25 + pos_file_path = `pwd` + router = double('router') + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, batch_size, pos_file_path, router, @tag) + + channel = double(Concurrent::Channel) + expect(Concurrent::Channel).to receive(:new).with(capacity: batch_size).and_return(channel) + + timer_task = double(Concurrent::TimerTask) + expect(Concurrent::TimerTask).to receive(:new).and_return timer_task + + expect(timer_task).to receive(:execute) + xray.violations(Date.today) + end + + end + + describe "#process" do + let(:violation1){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), + "watch_name": "watch1", + "issue_id": "55444", + "violation_details_url": "http://www.com"} + } + let(:violation2){ { "created": Date.parse(Date.today.to_s).strftime("%Y-%m-%dT%H:%M:%SZ"), + "watch_name": "watch2", + "issue_id": "55443", + "violation_details_url": "http://www.com"} + } + + let(:violations_channel) { Concurrent::Array.new } + + it "skips processed violation" do + pos_file_path = `pwd` + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router, @tag) + + violations_channel << violation1 + + pos_file = double(PositionFile) + allow(PositionFile).to receive(:new).and_return pos_file + allow(pos_file).to receive(:processed?).and_return true + + xray.process(violation1, violations_channel) + + expect(violations_channel.size).to eq 1 + end + + it "adds unprocessed violation to the channel" do + pos_file_path = `pwd` + xray = Xray.new(@jpd_url, @username, @apikey, @wait_interval, @batch_size, @pos_file_path, @router, @tag) + + violations_channel << violation1 + + pos_file = double(PositionFile) + allow(PositionFile).to receive(:new).and_return pos_file + allow(pos_file).to receive(:processed?).and_return false + + xray.process(violation2, violations_channel) + + expect(violations_channel.size).to eq 2 + end + end + +end \ No newline at end of file diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/helper.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/helper.rb index 1562063..1959444 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/helper.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/helper.rb @@ -4,5 +4,6 @@ require "fluent/test/driver/input" require "fluent/test/helpers" + Test::Unit::TestCase.include(Fluent::Test::Helpers) Test::Unit::TestCase.extend(Fluent::Test::Helpers) diff --git a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb index 5ca271b..a635395 100644 --- a/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb +++ b/fluentd/plugins/input/fluent-plugin-jfrog-siem/test/plugin/test_in_jfrog_siem.rb @@ -12,12 +12,15 @@ class JfrogSiemInputTest < Test::Unit::TestCase # Default configuration for tests CONFIG = %[ - tag "test_tag" - jpd_url JPD_URL - username USER - apikey API_KEY - pos_file "test_pos.txt" - ] + tag "jfrog.xray.siem.vulnerabilities" + jpd_url "JPDURL" + username "admin" + apikey "APIKEY" + pos_file_path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/" + wait_interval 10 + from_date "2016-01-01" + batch_size 25 + ] private @@ -28,7 +31,11 @@ def create_driver(conf = CONFIG) sub_test_case 'Testing' do test 'Testing plugin in_jfrog_siem' do d = create_driver(CONFIG) - d.run + begin + d.run + rescue => e + raise "Test failed due to #{e}" + end end end end