From a080b7cc824ae77cfde1e1dac92784a62b3f25d6 Mon Sep 17 00:00:00 2001 From: Guilherme Santos Date: Wed, 3 Dec 2025 12:11:26 +0100 Subject: [PATCH 1/2] GRIF-524 - Refactor of integration-e2e to run on github CI --- .gitignore | 4 ++ docker-compose.lcm.yml | 19 ++++--- gooddata.gemspec | 9 +++- .../bricks/middleware/aws_middleware.rb | 1 + lib/gooddata/rest/connection.rb | 2 + spec/environment/environment.rb | 1 + .../params/provisioning_brick.json.erb | 7 ++- .../integration/params/release_brick.json.erb | 3 +- ...e_brick_delete_old_master_project.json.erb | 3 +- .../release_brick_set_master_project.json.erb | 19 +++++-- .../spec/shared_contexts_for_lcm.rb | 4 +- spec/lcm/integration/support/in_memory_ads.rb | 53 +++++++++++++++---- spec/lcm/integration/support/s3_helper.rb | 13 +++-- spec/vcr_configurer.rb | 8 ++- 14 files changed, 105 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 53678c00b..242a8d517 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,7 @@ deprecations.txt # test cache files spec/cache/ + +# local/ci execution +lib/gooddata/cloud_resources/*/*/*.jar +ci/*/target/ diff --git a/docker-compose.lcm.yml b/docker-compose.lcm.yml index 64d786962..74474fa7e 100644 --- a/docker-compose.lcm.yml +++ b/docker-compose.lcm.yml @@ -1,12 +1,10 @@ -version: '2' services: appstore: image: gooddata/appstore build: context: . dockerfile: Dockerfile.jruby - links: - - localstack + working_dir: /src environment: - GD_ENV - GDC_LOG_LEVEL @@ -57,16 +55,17 @@ services: - GD_STG_DEFAULT_PASSWORD volumes: - .:/src - volumes_from: - - bundle - mem_limit: 2500m + - bundle_cache:/src/bundle + deploy: + resources: + limits: + memory: 2500m localstack: image: hahihula/localstack_dockerfile expose: - "4572" environment: - SERVICES=s3:4572 - bundle: - image: busybox - volumes: - - /bundle + +volumes: + bundle_cache: diff --git a/gooddata.gemspec b/gooddata.gemspec index d622276e4..c01b8ba42 100644 --- a/gooddata.gemspec +++ b/gooddata.gemspec @@ -56,7 +56,7 @@ Gem::Specification.new do |s| s.add_dependency 'unf', '~> 0.1.4' end s.add_development_dependency 'simplecov', '~> 0.12' - s.add_development_dependency 'webmock', '~> 2.3.1' + s.add_development_dependency 'webmock', '~> 3.26.0' s.add_development_dependency 'yard', '~> 0.9.11' s.add_development_dependency 'yard-rspec', '~> 0.1' s.add_development_dependency 'pry' @@ -68,7 +68,12 @@ Gem::Specification.new do |s| s.add_development_dependency 'vcr', '5.0.0' s.add_development_dependency 'hashdiff', '~> 0.4' - s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java' + if RUBY_PLATFORM == 'java' + s.add_development_dependency 'activerecord-jdbcsqlite3-adapter' + s.add_development_dependency 'activerecord' + else + s.add_development_dependency 'sqlite3' + end if RUBY_VERSION >= '2.8' s.add_dependency 'activesupport', '>= 6.0.3.1' diff --git a/lib/gooddata/bricks/middleware/aws_middleware.rb b/lib/gooddata/bricks/middleware/aws_middleware.rb index 93bfddbe0..3bacf0020 100644 --- a/lib/gooddata/bricks/middleware/aws_middleware.rb +++ b/lib/gooddata/bricks/middleware/aws_middleware.rb @@ -60,6 +60,7 @@ def rewrite_for_aws_sdk_v2(config) 'SDK uses SSL everywhere. To disable SSL you must ' \ 'configure an endpoint that uses http://.' end + config.delete('endpoint') if config['endpoint'].nil? || config['endpoint'].to_s.strip.empty? config end end diff --git a/lib/gooddata/rest/connection.rb b/lib/gooddata/rest/connection.rb index 4c1a3642e..258854720 100644 --- a/lib/gooddata/rest/connection.rb +++ b/lib/gooddata/rest/connection.rb @@ -177,6 +177,8 @@ def initialize(opts) # Connect using username and password def connect(username, password, options = {}) + Psych::Parser.code_point_limit = 100_000_000 + server = options[:server] || Helpers::AuthHelper.read_server options = DEFAULT_LOGIN_PAYLOAD.merge(options) headers = options[:headers] || {} diff --git a/spec/environment/environment.rb b/spec/environment/environment.rb index 4a41ff9b9..e57773cad 100644 --- a/spec/environment/environment.rb +++ b/spec/environment/environment.rb @@ -49,6 +49,7 @@ def initial_secrets(env) s3_bucket_name: ENV['RT_S3_BUCKET_NAME'], s3_access_key_id: ENV['RT_S3_ACCESS_KEY'], s3_secret_access_key: ENV['RT_S3_SECRET_KEY'], + s3_session_token: ENV['RT_S3_SESSION_TOKEN'], redshift_password: ENV['REDSHIFT_PASSWORD'], redshift_access_key: ENV['REDSHIFT_ACCESS_KEY'], redshift_secret_key: ENV['REDSHIFT_SECRET_KEY'], diff --git a/spec/lcm/integration/params/provisioning_brick.json.erb b/spec/lcm/integration/params/provisioning_brick.json.erb index 94eabddc7..05bf00f12 100644 --- a/spec/lcm/integration/params/provisioning_brick.json.erb +++ b/spec/lcm/integration/params/provisioning_brick.json.erb @@ -36,7 +36,9 @@ "aws_client": { "access_key_id": "${s3_access_key}", "secret_access_key": "${s3_secret_access_key}", - "endpoint": "<%= s3_endpoint %>" + "session_token": "${s3_session_token}", + "endpoint": "<%= s3_endpoint %>", + "region": "<%= s3_region %>" } }, "gd_encoded_hidden_params": { @@ -53,5 +55,6 @@ "GDC_USERNAME": "<%= config[:username] %>", "GDC_PASSWORD": "<%= config[:password] %>", "s3_secret_access_key": "<%= s3_secret_access_key %>", - "s3_access_key": "<%= s3_access_key %>" + "s3_access_key": "<%= s3_access_key %>", + "s3_session_token": "<%= s3_session_token %>" } diff --git a/spec/lcm/integration/params/release_brick.json.erb b/spec/lcm/integration/params/release_brick.json.erb index c2424ce51..dce996571 100644 --- a/spec/lcm/integration/params/release_brick.json.erb +++ b/spec/lcm/integration/params/release_brick.json.erb @@ -10,8 +10,7 @@ "SEGMENTS": <%= segments %>, "DATA_PRODUCT": "<%= data_product %>", "TOKENS": { - "pg": "<%= config[:prod_token] %>", - "vertica": "<%= config[:vertica_prod_token] %>" + "pg": "<%= config[:prod_token] %>" }, <% if defined? jdbc_url %> "ads_client": { diff --git a/spec/lcm/integration/params/release_brick_delete_old_master_project.json.erb b/spec/lcm/integration/params/release_brick_delete_old_master_project.json.erb index 316b6c91c..05ee6ec9f 100644 --- a/spec/lcm/integration/params/release_brick_delete_old_master_project.json.erb +++ b/spec/lcm/integration/params/release_brick_delete_old_master_project.json.erb @@ -6,8 +6,7 @@ "SEGMENTS": <%= segments %>, "DATA_PRODUCT": "<%= data_product %>", "TOKENS": { - "pg": "<%= config[:prod_token] %>", - "vertica": "<%= config[:vertica_prod_token] %>" + "pg": "<%= config[:prod_token] %>" }, <% if defined? jdbc_url %> "ads_client": { diff --git a/spec/lcm/integration/params/release_brick_set_master_project.json.erb b/spec/lcm/integration/params/release_brick_set_master_project.json.erb index 746a52802..05ee6ec9f 100644 --- a/spec/lcm/integration/params/release_brick_set_master_project.json.erb +++ b/spec/lcm/integration/params/release_brick_set_master_project.json.erb @@ -3,6 +3,11 @@ "CLIENT_GDC_PROTOCOL": "https", "CLIENT_GDC_HOSTNAME": "<%= config[:prod_server] %>", "gd_encoded_params": { + "SEGMENTS": <%= segments %>, + "DATA_PRODUCT": "<%= data_product %>", + "TOKENS": { + "pg": "<%= config[:prod_token] %>" + }, <% if defined? jdbc_url %> "ads_client": { "username": "${GDC_USERNAME}", @@ -10,13 +15,21 @@ "jdbc_url": "<%= jdbc_url %>" }, <% end %> - "SEGMENTS": <%= segments %>, - "DATA_PRODUCT": "<%= data_product %>" + "technical_user": [ + "${GDC_USERNAME}" + ], + "development_client": { + "protocol": "https", + "hostname": "<%= config[:dev_server] %>", + "username": "${GDC_USERNAME}", + "password": "${GDC_PASSWORD}" + } }, + "production_tag": "dashboard, metric", "release_table_name": "<%= release_table_name %>", "fail_early": "true", "strict": "true", "GDC_USERNAME": "<%= config[:username] %>", "GDC_PASSWORD": "<%= config[:password] %>", - "set_master_project": "<%= master_project_id %>" + "GDC_LOG_LEVEL": "DEBUG" } diff --git a/spec/lcm/integration/spec/shared_contexts_for_lcm.rb b/spec/lcm/integration/spec/shared_contexts_for_lcm.rb index 14034e0c0..cb3ea57f6 100644 --- a/spec/lcm/integration/spec/shared_contexts_for_lcm.rb +++ b/spec/lcm/integration/spec/shared_contexts_for_lcm.rb @@ -123,7 +123,9 @@ def create_workspace_csv(workspaces, client_id_column) data = { segment_id: "LCM_SPEC_#{segment}_#{i}", development_pid: @project.obj_id, - driver: segment == 'PREMIUM' ? 'vertica' : 'pg', + # TODO: Uncomment below after https://gooddata.atlassian.net/browse/GRIF-655 is fixed + # driver: segment == 'PREMIUM' ? 'vertica' : 'pg', + driver: 'pg', master_name: "LCM spec master project (#{segment} #{i}) " + '##{version}' # rubocop:disable Lint/InterpolationCheck } diff --git a/spec/lcm/integration/support/in_memory_ads.rb b/spec/lcm/integration/support/in_memory_ads.rb index 57348cb4f..af9a0a5a7 100644 --- a/spec/lcm/integration/support/in_memory_ads.rb +++ b/spec/lcm/integration/support/in_memory_ads.rb @@ -1,11 +1,26 @@ -require 'sqlite3' unless RUBY_PLATFORM == 'java' +if RUBY_PLATFORM == 'java' + require 'active_record' + require 'activerecord-jdbcsqlite3-adapter' +else + require 'sqlite3' +end module Support class InMemoryAds def initialize - db = SQLite3::Database.new(':memory:') - db.results_as_hash = true - @db = db + if RUBY_PLATFORM == 'java' + # Use ActiveRecord with JDBC adapter for JRuby + ActiveRecord::Base.establish_connection( + adapter: 'sqlite3', + database: ':memory:' + ) + @db = ActiveRecord::Base.connection + else + # Use sqlite3 gem for MRI + db = SQLite3::Database.new(':memory:') + db.results_as_hash = true + @db = db + end end def data @@ -33,12 +48,30 @@ def execute(*args, &block) end def execute_with_headers(*args) - res = @db.execute(*args) - res.map do |row| - # sqlite3 returns hash with both column names and numbers, we want only names - res = GoodData::Helpers.symbolize_keys(row.reject { |k, _| k.is_a? Integer }) - yield res if block_given? - res + if RUBY_PLATFORM == 'java' + # ActiveRecord JDBC adapter + result = @db.exec_query(*args) + # ActiveRecord returns arrays of arrays, convert to hash format + columns = result.columns if result.respond_to?(:columns) + result.map do |row| + if row.is_a?(Array) && columns + # Convert array to hash + row_hash = columns.zip(row).to_h + res = GoodData::Helpers.symbolize_keys(row_hash) + else + res = GoodData::Helpers.symbolize_keys(row.is_a?(Hash) ? row : {}) + end + yield res if block_given? + res + end + else + res = @db.execute(*args) + res.map do |row| + # sqlite3 returns hash with both column names and numbers, we want only names + res = GoodData::Helpers.symbolize_keys(row.reject { |k, _| k.is_a? Integer }) + yield res if block_given? + res + end end end diff --git a/spec/lcm/integration/support/s3_helper.rb b/spec/lcm/integration/support/s3_helper.rb index 35b66120d..625f353fa 100644 --- a/spec/lcm/integration/support/s3_helper.rb +++ b/spec/lcm/integration/support/s3_helper.rb @@ -1,10 +1,9 @@ module Support class S3Helper LOCALSTACK_ENDPOINT = 'http://localstack:4572'.freeze - S3_ENDPOINT = 'https://s3.amazonaws.com'.freeze USER_FILTERS_KEY = 'user_filters' USERS_KEY = 'users_brick_input' - REGION = 'us-east-1' + REGION = 'eu-west-1' class << self include GoodData::Environment::ConnectionHelper @@ -13,16 +12,14 @@ def upload_file(file, object_name) bucket_name = SECRETS[:s3_bucket_name] s3_endpoint = if GoodData::Environment::LOCALSTACK Support::S3Helper::LOCALSTACK_ENDPOINT - else - Support::S3Helper::S3_ENDPOINT end s3 = Aws::S3::Resource.new( access_key_id: SECRETS[:s3_access_key_id], secret_access_key: SECRETS[:s3_secret_access_key], + session_token: SECRETS[:s3_session_token], endpoint: s3_endpoint, region: REGION, - force_path_style: true ) bucket = s3.bucket(bucket_name) @@ -32,9 +29,11 @@ def upload_file(file, object_name) { s3_bucket: bucket_name, - s3_endpoint: s3_endpoint, + s3_endpoint: s3_endpoint ? bucket.url : s3_endpoint, s3_access_key: SECRETS[:s3_access_key_id], - s3_secret_access_key: SECRETS[:s3_secret_access_key] + s3_secret_access_key: SECRETS[:s3_secret_access_key], + s3_session_token: SECRETS[:s3_session_token], + s3_region: REGION } end end diff --git a/spec/vcr_configurer.rb b/spec/vcr_configurer.rb index 2577512af..da43ff692 100644 --- a/spec/vcr_configurer.rb +++ b/spec/vcr_configurer.rb @@ -43,8 +43,12 @@ def self.setup vcr_config.allow_http_connections_when_no_cassette = true vcr_config.configure_rspec_metadata! - vcr_config.ignore_request do - @ignore_vcr_requests + vcr_config.ignore_request do |request| + @ignore_vcr_requests || + request.uri.include?('s3.amazonaws.com') || + request.uri.match?(/s3[.-][a-z0-9-]+\.amazonaws\.com/) || + request.uri.include?('staging-lcm-prod.intgdc.com') || + request.uri.include?('staging-lcm-dev.intgdc.com') end vcr_config.default_cassette_options = { From c477475ef922ceb5c4d108e050fdca91d2e32966 Mon Sep 17 00:00:00 2001 From: Guilherme Santos Date: Fri, 23 Jan 2026 15:59:23 +0100 Subject: [PATCH 2/2] bump webmock only on ruby > 2.6 --- gooddata.gemspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gooddata.gemspec b/gooddata.gemspec index c01b8ba42..536261c3f 100644 --- a/gooddata.gemspec +++ b/gooddata.gemspec @@ -35,6 +35,7 @@ Gem::Specification.new do |s| s.add_development_dependency 'rspec', '~> 3.12.0' s.add_development_dependency 'rspec-expectations', '~> 3.12' s.add_development_dependency 'rspec_junit_formatter', '~> 0.6.0' + s.add_development_dependency 'webmock', '~> 3.26.0' s.add_dependency 'azure-storage-blob', '~> 2.0' s.add_dependency 'nokogiri', '~> 1', '>= 1.10.8' @@ -56,7 +57,7 @@ Gem::Specification.new do |s| s.add_dependency 'unf', '~> 0.1.4' end s.add_development_dependency 'simplecov', '~> 0.12' - s.add_development_dependency 'webmock', '~> 3.26.0' + s.add_development_dependency 'webmock', '~> 2.3.1' s.add_development_dependency 'yard', '~> 0.9.11' s.add_development_dependency 'yard-rspec', '~> 0.1' s.add_development_dependency 'pry'