From a393193e130bf996e3345efb8313b5d516ddfb2f Mon Sep 17 00:00:00 2001 From: Alex Hung Date: Wed, 24 Feb 2021 11:28:22 -0800 Subject: [PATCH 1/8] Remove unused QsS3* parameters --- ...artifactory-ec2-existing-vpc.template.yaml | 47 +------------------ ...rog-artifactory-ec2-instance.template.yaml | 13 ----- .../jfrog-xray-ec2-instance.template.yaml | 27 +++-------- 3 files changed, 9 insertions(+), 78 deletions(-) diff --git a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml index bf67044..f5dd1a0 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml @@ -61,11 +61,6 @@ Metadata: - DatabaseInstance - DatabaseAllocatedStorage - MultiAzDatabase - - Label: - default: AWS Quick Start configuration - Parameters: - - QsS3BucketName - - QsS3KeyPrefix - Label: default: JFrog Xray Configuration Parameters: @@ -152,16 +147,8 @@ Metadata: default: Database allocated storage MultiAzDatabase: default: High-availability database - QsS3BucketName: - default: Quick Start S3 bucket name - QsS3KeyPrefix: - default: Quick Start S3 key prefix - QsS3BucketRegion: - default: Quick Start S3 bucket region InstallXray: default: Install JFrog Xray - XrayVersion: - default: Version of Xray to install XrayNumberOfInstances: default: Number of JFrog Xray instances XrayInstanceType: @@ -173,6 +160,8 @@ Metadata: Parameters: AvailabilityZones: Description: List of Availability Zones to use for the subnets in the VPC. Two + XrayVersion: + default: Version of Xray to install Availability Zones are used for this deployment. Type: List KeyPairName: @@ -461,28 +450,6 @@ Parameters: - "false" Default: "true" Type: String - QsS3BucketName: - Description: S3 bucket name for the Quick Start assets. This string can include - numbers, lowercase letters, and hyphens (-). It cannot start - or end with a hyphen (-). - AllowedPattern: ^[0-9a-z]+([0-9a-z-]*[0-9a-z])*$ - ConstraintDescription: Quick Start bucket name can include numbers, lowercase - letters, and hyphens (-). It cannot start or end with a hyphen (-). - Default: aws-quickstart - Type: String - QsS3KeyPrefix: - Description: S3 key prefix for the Quick Start assets. Quick Start key prefix - can include numbers, lowercase letters, uppercase letters, hyphens (-), and - forward slash (/). - AllowedPattern: ^[0-9a-zA-Z-/]*$ - ConstraintDescription: Quick Start key prefix can include numbers, lowercase letters, - uppercase letters, hyphens (-), and forward slash (/). - Default: quickstart-jfrog-artifactory/ - Type: String - QsS3BucketRegion: - Default: 'us-east-1' - Description: AWS Region where the Quick Start S3 bucket (QSS3BucketName) is hosted. If you use your own bucket, you must specify your own value. - Type: String InstallXray: Description: Choose true to install JFrog Xray instance(s). ConstraintDescription: True or False @@ -824,10 +791,6 @@ Resources: MaxScalingNodes: '1' # Always have 1 MasterNode DeploymentTag: !If [IsArtifactory, "ArtifactoryPrimary", "JcrPrimary"] HostRole: !Ref ArtifactoryHostRole - QsS3BucketName: !Ref QsS3BucketName - QsS3KeyPrefix: !Ref QsS3KeyPrefix - QsS3Uri: !Sub https://${QsS3BucketName}.s3.${AWS::URLSuffix}/${QsS3KeyPrefix} - AmiId: !Join ['', !Split [".", !Ref ArtifactoryVersion]] ArtifactoryProduct: !Ref ArtifactoryProduct ArtifactoryLicensesSecretName: !Ref SmLicenseName ArtifactoryServerName: !Ref ArtifactoryServerName @@ -898,9 +861,6 @@ Resources: TargetGroupARN: !Ref ArtifactoryTargetGroup SSLTargetGroupARN: !Ref ArtifactorySslTargetGroup InternalTargetGroupARN: !Ref ArtifactoryInternalTargetGroup - QsS3BucketName: !Ref QsS3BucketName - QsS3KeyPrefix: !Ref QsS3KeyPrefix - QsS3Uri: !Sub https://${QsS3BucketName}.s3.${AWS::URLSuffix}/${QsS3KeyPrefix} XrayHostRole: Condition: EnableXray Type: AWS::IAM::Role @@ -972,9 +932,6 @@ Resources: MinScalingNodes: !Ref XrayNumberOfInstances MaxScalingNodes: !Ref XrayNumberOfInstances DeploymentTag: 'xray' - QsS3BucketName: !Ref QsS3BucketName - QsS3KeyPrefix: !Ref QsS3KeyPrefix - QsS3Uri: !Sub https://${QsS3BucketName}.s3.${AWS::URLSuffix}/${QsS3KeyPrefix} DatabaseDriver: !GetAtt ArtifactoryCoreInfraStack.Outputs.DatabaseDriver DatabaseType: !GetAtt ArtifactoryCoreInfraStack.Outputs.DatabaseType DatabaseUser: !Ref DatabaseUser diff --git a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml index 569d471..78983e9 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml @@ -15,12 +15,6 @@ Parameters: Type: String ArtifactoryProduct: Type: String - QsS3BucketName: - Type: String - QsS3KeyPrefix: - Type: String - QsS3Uri: - Type: String ArtifactoryLicensesSecretName: Type: String ArtifactoryServerName: @@ -164,13 +158,6 @@ Resources: ArtifactoryLaunchConfiguration: Type: 'AWS::AutoScaling::LaunchConfiguration' Metadata: - AWS::CloudFormation::Authentication: - S3AccessCreds: - type: S3 - roleName: - - !Ref HostRole # !Ref ArtifactoryHostRole - buckets: - - !Ref QsS3BucketName AWS::CloudFormation::Init: configSets: artifactory_install: diff --git a/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml index 31f6a74..4190aa0 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml @@ -13,12 +13,6 @@ Parameters: Type: Number DeploymentTag: Type: String - QsS3BucketName: - Type: String - QsS3KeyPrefix: - Type: String - QsS3Uri: - Type: String DatabaseDriver: Type: String DatabaseType: @@ -129,13 +123,6 @@ Resources: XrayLaunchConfiguration: Type: AWS::AutoScaling::LaunchConfiguration Metadata: - AWS::CloudFormation::Authentication: - S3AccessCreds: - type: S3 - roleName: - - !Ref XrayHostRole - buckets: - - !Ref QsS3BucketName AWS::CloudFormation::Init: configSets: xray_install: @@ -235,13 +222,6 @@ Resources: } - S3URI=${QsS3Uri} - - yum update --security -y &> /var/log/userdata.yum_security_update.log - - yum install -y git python3 libselinux-python3 - yum install -y postgresql-server postgresql-devel - echo $PATH PATH=/opt/aws/bin:$PATH @@ -255,6 +235,13 @@ Resources: pip install --upgrade pip pip install wheel + S3URI=${QsS3Uri} + + yum update --security -y &> /var/log/userdata.yum_security_update.log + + yum install -y git python3 libselinux-python3 + yum install -y postgresql-server postgresql-devel + # Install Cloudformation helper scripts pip install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz 2>&1 | tee /var/log/userdata.aws_cfn_bootstrap_install.log From afbe35aa3a2a05c4e0b7d4691e99116d73ddbdd2 Mon Sep 17 00:00:00 2001 From: Alex Hung Date: Wed, 24 Feb 2021 11:30:39 -0800 Subject: [PATCH 2/8] Replace DatabaseEngine param with hardcoded values --- ...ifactory-core-infrastructure.template.yaml | 84 ++++--------------- ...artifactory-ec2-existing-vpc.template.yaml | 10 --- 2 files changed, 18 insertions(+), 76 deletions(-) diff --git a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-core-infrastructure.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-core-infrastructure.template.yaml index 6f907c0..90f0ea8 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-core-infrastructure.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-core-infrastructure.template.yaml @@ -39,8 +39,6 @@ Parameters: - "true" - "false" Type: String - DatabaseEngine: - Type: String DatabaseUser: Type: String DatabasePassword: @@ -65,15 +63,6 @@ Parameters: Type: Number Mappings: - DatabaseMap: - Postgres: - Name: postgresql - DatabaseVersion: 11.5 - Driver: "org.postgresql.Driver" - Plugin: postgresql-42.2.9.jar - PluginURL: https://jdbc.postgresql.org/download/ - port: "5432" - extraDatabaseOps: "" ReleaseStageMap: BETA: ProDockerRepo: "earlyaccess-docker.jfrog.io/jfrog/artifactory-pro" @@ -229,11 +218,8 @@ Resources: Properties: AllocatedStorage: !Ref DatabaseAllocatedStorage MultiAZ: !Ref MultiAzDatabase - Engine: !Ref DatabaseEngine - EngineVersion: !FindInMap - - DatabaseMap - - !Ref DatabaseEngine - - DatabaseVersion + Engine: Postgres + EngineVersion: "11.5" MasterUsername: !Ref DatabaseUser MasterUserPassword: !Ref DatabasePassword DBInstanceClass: !Ref DatabaseInstance @@ -256,34 +242,16 @@ Resources: ToPort: 22 CidrIp: !Ref VpcCidr - IpProtocol: tcp - FromPort: !FindInMap - - DatabaseMap - - !Ref DatabaseEngine - - port - ToPort: !FindInMap - - DatabaseMap - - !Ref DatabaseEngine - - port + FromPort: 5432 + ToPort: 5432 CidrIp: !Ref PrivateSubnet1Cidr - IpProtocol: tcp - FromPort: !FindInMap - - DatabaseMap - - !Ref DatabaseEngine - - port - ToPort: !FindInMap - - DatabaseMap - - !Ref DatabaseEngine - - port + FromPort: 5432 + ToPort: 5432 CidrIp: !Ref PrivateSubnet2Cidr - IpProtocol: tcp - FromPort: !FindInMap - - DatabaseMap - - !Ref DatabaseEngine - - port - ToPort: !FindInMap - - DatabaseMap - - !Ref DatabaseEngine - - port + FromPort: 5432 + ToPort: 5432 CidrIp: !Ref PrivateSubnet3Cidr SecurityGroupEgress: - IpProtocol: tcp @@ -350,41 +318,25 @@ Outputs: Value: !Ref ArtifactoryS3Bucket Description: Actual S3 bucket created for Artifactory DatabaseDriver: - Value: !FindInMap [DatabaseMap, !Ref DatabaseEngine, Driver] + Value: "org.postgresql.Driver" DatabasePlugin: - Value: !FindInMap [DatabaseMap, !Ref DatabaseEngine, Plugin] + Value: postgresql-42.2.9.jar DatabasePluginUrl: - Value: !Sub - - "${MainURL}${PluginVersion}" - - { - MainURL: !FindInMap [DatabaseMap, !Ref DatabaseEngine, PluginURL], - PluginVersion: !FindInMap [DatabaseMap, !Ref DatabaseEngine, Plugin] - } + Value: https://jdbc.postgresql.org/download/postgresql-42.2.9.jar" DatabaseType: - Value: !FindInMap [DatabaseMap, !Ref DatabaseEngine, Name] + Value: postgresql DatabaseUrl: Value: !Sub - - "jdbc:${DatabaseType}://${ArtifactoryDatabaseEndpointAddress}:${port}/${DatabaseName}${extraDatabaseOps}" - - { - DatabaseType: !FindInMap [DatabaseMap, !Ref DatabaseEngine, Name], - ArtifactoryDatabaseEndpointAddress: !GetAtt ArtifactoryDatabase.Endpoint.Address, - port: !FindInMap [DatabaseMap, !Ref DatabaseEngine, port], - extraDatabaseOps: !FindInMap [DatabaseMap, !Ref DatabaseEngine, extraDatabaseOps], - } + - "jdbc:postgresql://${ArtifactoryDatabaseEndpointAddress}:5432/${DatabaseName}" + - ArtifactoryDatabaseEndpointAddress: !GetAtt ArtifactoryDatabase.Endpoint.Address XrayMasterDatabaseUrl: Value: !Sub - - "${ArtifactoryDatabaseEndpointAddress}:${port}/${DatabaseName}?sslmode=disable" - - { - ArtifactoryDatabaseEndpointAddress: !GetAtt ArtifactoryDatabase.Endpoint.Address, - port: !FindInMap [DatabaseMap, !Ref DatabaseEngine, port], - } + - "${ArtifactoryDatabaseEndpointAddress}:5432/${DatabaseName}?sslmode=disable" + - ArtifactoryDatabaseEndpointAddress: !GetAtt ArtifactoryDatabase.Endpoint.Address XrayDatabaseUrl: Value: !Sub - - "${ArtifactoryDatabaseEndpointAddress}:${port}/xraydb?sslmode=disable" - - { - ArtifactoryDatabaseEndpointAddress: !GetAtt ArtifactoryDatabase.Endpoint.Address, - port: !FindInMap [DatabaseMap, !Ref DatabaseEngine, port], - } + - "${ArtifactoryDatabaseEndpointAddress}:5432/xraydb?sslmode=disable" + - ArtifactoryDatabaseEndpointAddress: !GetAtt ArtifactoryDatabase.Endpoint.Address ProDockerRepo: Value: !FindInMap - ReleaseStageMap diff --git a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml index f5dd1a0..41ed4d8 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml @@ -55,7 +55,6 @@ Metadata: default: Amazon RDS configuration Parameters: - DatabaseName - - DatabaseEngine - DatabaseUser - DatabasePassword - DatabaseInstance @@ -135,8 +134,6 @@ Metadata: default: Default Java memory settings DatabaseName: default: Database name - DatabaseEngine: - default: Database engine DatabaseUser: default: Database user DatabasePassword: @@ -395,12 +392,6 @@ Parameters: ConstraintDescription: 1 to 60 alphanumeric characters First character must be a letter. Default: artdb Type: String - DatabaseEngine: - Description: Database engine that you want to run, which is currently locked to MySQL. - AllowedValues: - - Postgres - Default: Postgres - Type: String DatabaseUser: Description: Login ID for the master user of your database instance. MinLength: '1' @@ -582,7 +573,6 @@ Resources: SubnetIds: !Join [",", [!Ref PrivateSubnet1Id, !Ref PrivateSubnet2Id]] DatabaseAllocatedStorage: !Ref DatabaseAllocatedStorage MultiAzDatabase: !Ref MultiAzDatabase - DatabaseEngine: !Ref DatabaseEngine DatabaseUser: !Ref DatabaseUser DatabasePassword: !Ref DatabasePassword DatabaseInstance: !Ref DatabaseInstance From da13da97eaa53085526f9227f93ec84fed5a8adf Mon Sep 17 00:00:00 2001 From: Alex Hung Date: Wed, 24 Feb 2021 11:31:31 -0800 Subject: [PATCH 3/8] Remove bastion resources --- ...artifactory-ec2-existing-vpc.template.yaml | 158 ------------------ 1 file changed, 158 deletions(-) diff --git a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml index 41ed4d8..f4875af 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml @@ -10,7 +10,6 @@ Metadata: Parameters: - KeyPairName - AccessCidr - - RemoteAccessCidr - Label: default: Network configuration Parameters: @@ -24,16 +23,6 @@ Metadata: - PrivateSubnet1Cidr - PrivateSubnet2Cidr - ELBScheme - - Label: - default: Bastion configuration - Parameters: - - ProvisionBastionHost - - BastionInstanceType - - BastionOs - - BastionRootVolumeSize - - BastionEnableTcpForwarding - - NumBastionHosts - - BastionEnableX11Forwarding - Label: default: Amazon EC2 configuration Parameters: @@ -92,24 +81,8 @@ Metadata: default: Private subnet 2 CIDR AccessCidr: default: Permitted IP range - RemoteAccessCidr: - default: Remote access CIDR ELBScheme: default: Elastic Load Balancing scheme - ProvisionBastionHost: - default: Bastion instance - BastionInstanceType: - default: Bastion instance type - BastionRootVolumeSize: - default: Bastion root volume size - BastionEnableTcpForwarding: - default: Bastion enable TCP forwarding - BastionEnableX11Forwarding: - default: Bastion enable X11 forwarding - BastionOs: - default: Bastion operating system - NumBastionHosts: - default: Number of bastion instances VolumeSize: default: EBS root volume size InstanceType: @@ -205,12 +178,6 @@ Parameters: For example, you might want to grant only your corporate network access to the software. AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ Type: String - RemoteAccessCidr: - Description: Remote CIDR range that allows you to connect to the bastion instance by using SSH. - We recommend that you set this value to a trusted IP range. - For example, you might want to grant specific ranges inside your corporate network SSH access. - AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$ - Type: String ELBScheme: Description: Choose whether this is internet facing or internal. AllowedValues: @@ -218,68 +185,6 @@ Parameters: - internet-facing Default: internet-facing Type: String - ProvisionBastionHost: - Description: Choose Disabled to skip creating a bastion instance. Due to the JFrog Container Registry nodes being - created in private subnets, the default setting of Enabled this is highly recommended. - AllowedValues: - - "Enabled" - - "Disabled" - Default: "Enabled" - Type: String - BastionInstanceType: - Description: Size of the bastion instances. - AllowedValues: - - t3.nano - - t3.micro - - t3.small - - t3.medium - - t3.large - - m5.large - - m5.xlarge - - m5.2xlarge - - m5.4xlarge - Default: "t3.micro" - Type: String - BastionRootVolumeSize: - Description: Size of the root volume on the bastion instances. - Default: 10 - Type: Number - BastionEnableTcpForwarding: - Description: Choose whether to enable TCPForwarding via the bootstrapping of the bastion instance - or not. - AllowedValues: - - "true" - - "false" - Default: "true" - Type: String - BastionEnableX11Forwarding: - Description: Choose true to enable X11 via the bootstrapping of the bastion host. - Setting this value to true will enable X Windows over SSH. - X11 forwarding can be useful, but it is also a security risk, so it's recommended - that you keep the default (false) setting. - AllowedValues: - - "true" - - "false" - Default: "false" - Type: String - BastionOs: - Description: Linux distribution for the Amazon Machine Image (AMI) to be used for the bastion instances. - AllowedValues: - - "Amazon-Linux2-HVM" - - "CentOS-7-HVM" - - "Ubuntu-Server-20.04-LTS-HVM" - - "SUSE-SLES-15-HVM" - Default: "Amazon-Linux2-HVM" - Type: String - NumBastionHosts: - Description: Number of bastion instances to create. - AllowedValues: - - '1' - - '2' - - '3' - - '4' - Default: '1' - Type: String VolumeSize: Description: Size in gigabytes of the available storage (min 10GB); the Quick Start will create an Amazon Elastic Block Store (Amazon EBS) volumes of this size. @@ -491,7 +396,6 @@ Parameters: Type: String Conditions: - EnableBastion: !Equals [!Ref 'ProvisionBastionHost', 'Enabled'] IsArtifactory: !Not [!Equals [!Ref ArtifactoryProduct, 'JFrog-Container-Registry']] HasSecondaryNodes: !Not [!Equals [!Ref NumberOfSecondary, '0']] DefaultJava: !Equals [!Ref DefaultJavaMemSettings, "true"] @@ -499,63 +403,6 @@ Conditions: SmCertNameExists: !Not [!Equals [!Ref 'SmCertName', '']] Resources: - BastionRole: - Condition: EnableBastion - Type: "AWS::IAM::Role" - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: ec2.amazonaws.com - Action: sts:AssumeRole - Policies: - - PolicyName: QSBucketAccess - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: s3:GetObject - Resource: !Sub "arn:${AWS::Partition}:s3:::${QsS3BucketName}/*" - - Effect: Allow - Action: - - logs:CreateLogStream - - logs:GetLogEvents - - logs:PutLogEvents - - logs:DescribeLogGroups - - logs:DescribeLogStreams - - logs:PutRetentionPolicy - - logs:PutMetricFilter - - logs:CreateLogGroup - Resource: !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:*:*" - - Effect: Allow - Action: - - ec2:AssociateAddress - - ec2:DescribeAddresses - Resource: "*" - BastionStack: - Condition: EnableBastion - Type: AWS::CloudFormation::Stack - Properties: - TemplateURL: https://aws-quickstart.s3.amazonaws.com/quickstart-jfrog-artifactory/submodules/quickstart-linux-bastion/templates/linux-bastion.template - Parameters: - VPCID: !Ref VpcId - PublicSubnet1ID: !Ref PublicSubnet1Id - PublicSubnet2ID: !Ref PublicSubnet2Id - KeyPairName: !Ref KeyPairName - QSS3BucketName: !Ref QsS3BucketName - QSS3KeyPrefix: !Sub '${QsS3KeyPrefix}submodules/quickstart-linux-bastion/' - QSS3BucketRegion: !Ref QsS3BucketRegion - RemoteAccessCIDR: !Ref RemoteAccessCidr - BastionInstanceType: !Ref BastionInstanceType - RootVolumeSize: !Ref BastionRootVolumeSize - BastionAMIOS: !Ref BastionOs - EnableTCPForwarding: !Ref BastionEnableTcpForwarding - EnableX11Forwarding: !Ref BastionEnableX11Forwarding - AlternativeIAMRole: !Ref BastionRole - NumBastionHosts: !Ref NumBastionHosts - ArtifactoryCoreInfraStack: Type: AWS::CloudFormation::Stack Properties: @@ -981,11 +828,6 @@ Outputs: Value: !Ref ArtifactoryEc2Sg Export: Name: !Sub '${AWS::StackName}-ArtifactoryEc2Sg' - BastionIp: - Value: !If - - EnableBastion - - !GetAtt BastionStack.Outputs.EIP1 - - "" XrayMasterDatabaseUrl: Description: Database driver Value: !GetAtt ArtifactoryCoreInfraStack.Outputs.XrayMasterDatabaseUrl From 27a354577135af65c3ef968185162e13a3c21c61 Mon Sep 17 00:00:00 2001 From: Alex Hung Date: Wed, 24 Feb 2021 11:32:34 -0800 Subject: [PATCH 4/8] Replace RT and Xray version params with hardcoded values --- ...artifactory-ec2-existing-vpc.template.yaml | 31 ++++--------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml index f4875af..9e6e270 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml @@ -32,7 +32,6 @@ Metadata: default: JFrog Artifactory configuration Parameters: - ArtifactoryProduct - - ArtifactoryVersion - NumberOfSecondary - SmLicenseName - SmCertName @@ -53,7 +52,6 @@ Metadata: default: JFrog Xray Configuration Parameters: - InstallXray - - XrayVersion - XrayNumberOfInstances - XrayInstanceType - XrayDatabaseUser @@ -91,8 +89,6 @@ Metadata: default: Secondary instances ArtifactoryProduct: default: Artifactory product to install - ArtifactoryVersion: - default: Artifactory version SmLicenseName: default: Artifactory licenses secret name SmCertName: @@ -130,8 +126,6 @@ Metadata: Parameters: AvailabilityZones: Description: List of Availability Zones to use for the subnets in the VPC. Two - XrayVersion: - default: Version of Xray to install Availability Zones are used for this deployment. Type: List KeyPairName: @@ -245,14 +239,6 @@ Parameters: - JFrog-Container-Registry Default: JFrog-Artifactory-Pro Type: String - ArtifactoryVersion: - Description: Version of Artifactory that you want to deploy into the Quick Start. - See the release notes to select the version you want to deploy at - https://www.jfrog.com/confluence/display/RTF/Release+Notes. - AllowedPattern: ^(([0-9]|[1-9][0-9])\.){2}([1-9][0-9]|[0-9])$ - ConstraintDescription: A version that matches X.X.X per Artifactory releases - Default: 7.15.3 - Type: String SmLicenseName: Description: Secret name created in AWS Secrets Manager, which contains the Artifactory licenses. Default: '' @@ -354,12 +340,6 @@ Parameters: - "false" Default: "true" Type: String - XrayVersion: - Description: The version of Xray that you want to deploy into the Quick Start. - AllowedPattern: ^(([0-9]|[1-9][0-9])\.){2}([1-9][0-9]|[0-9])$ - ConstraintDescription: A version that matches X.X.X per Xray releases. - Default: 3.17.4 - Type: String XrayNumberOfInstances: Description: The number of Xray instances servers to complete your HA deployment. The minimum number is one; the maximum is seven. @@ -646,7 +626,8 @@ Resources: ArtifactoryPrimary: true MasterKey: !Ref MasterKey ExtraJavaOptions: !If [DefaultJava, !Sub "${ArtifactoryCoreInfraStack.Outputs.JavaOpts} ${ExtraJavaOptions}", !Ref ExtraJavaOptions] - ArtifactoryVersion: !Ref ArtifactoryVersion + AmiId: "7153" + ArtifactoryVersion: "7.15.3" KeyPairName: !Ref KeyPairName HostProfile: !Ref ArtifactoryHostProfile SecurityGroups: !Ref ArtifactoryEc2Sg @@ -669,7 +650,6 @@ Resources: MaxScalingNodes: !Ref NumberOfSecondary DeploymentTag: ArtifactorySecondary HostRole: !Ref ArtifactoryHostRole - AmiId: !Join ['', !Split [".", !Ref ArtifactoryVersion]] ArtifactoryProduct: !Ref ArtifactoryProduct ArtifactoryLicensesSecretName: !Ref SmLicenseName ArtifactoryServerName: !Ref ArtifactoryServerName @@ -688,7 +668,8 @@ Resources: ArtifactoryPrimary: false MasterKey: !Ref MasterKey ExtraJavaOptions: !If [DefaultJava, !Sub "${ArtifactoryCoreInfraStack.Outputs.JavaOpts} ${ExtraJavaOptions}", !Ref ExtraJavaOptions] - ArtifactoryVersion: !Ref ArtifactoryVersion + AmiId: "7153" + ArtifactoryVersion: "7.15.3" KeyPairName: !Ref KeyPairName HostProfile: !Ref ArtifactoryHostProfile SecurityGroups: !Ref ArtifactoryEc2Sg @@ -783,8 +764,8 @@ Resources: XrayDatabasePassword: !Ref XrayDatabasePassword XrayMasterDatabaseUrl: !GetAtt ArtifactoryCoreInfraStack.Outputs.XrayMasterDatabaseUrl XrayDatabaseUrl: !GetAtt ArtifactoryCoreInfraStack.Outputs.XrayDatabaseUrl - XrayVersion: !Ref XrayVersion - XrayAmiId: !Join ['', !Split [".", !Ref XrayVersion]] + XrayVersion: "3.17.4" + XrayAmiId: "3174" XrayHostRole: !Ref XrayHostRole XrayHostProfile: !Ref XrayHostProfile Outputs: From 9634e2a0477fefd7837d5a29dfc7d80a2d6edf4d Mon Sep 17 00:00:00 2001 From: Alex Hung Date: Wed, 24 Feb 2021 11:33:52 -0800 Subject: [PATCH 5/8] Remove all unnecessary steps in user scripts --- ...rog-artifactory-ec2-instance.template.yaml | 33 +++---------------- .../jfrog-xray-ec2-instance.template.yaml | 26 +-------------- 2 files changed, 5 insertions(+), 54 deletions(-) diff --git a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml index 78983e9..4e37adc 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml @@ -329,17 +329,9 @@ Resources: } - S3URI=${QsS3Uri} - - # Update OS - yum update -y - - # Install git - yum install -y epel-release git policycoreutils-python - - yum update --security -y 2>&1 | tee /var/log/userdata.yum_security_update.log - - yum install -y jq python3 libselinux-python3 + # Install jq + yum install -y epel-release + yum install -y jq echo $PATH @@ -347,24 +339,9 @@ Resources: echo $PATH - # Create virtual env and activate - python3 -m venv ~/venv --system-site-packages + # Activate virtual env source ~/venv/bin/activate - pip install --upgrade pip - pip install jmespath wheel - - # Install Cloudformation helper scripts - pip install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz 2>&1 | tee /var/log/userdata.aws_cfn_bootstrap_install.log - - pip install awscli 2>&1 | tee /var/log/userdata.awscli_install.log - - pip install ansible 2>&1 | tee /var/log/userdata.ansible_install.log - - mkdir ~/.jfrog_ami - - aws s3 --region ${AWS::Region} sync s3://${QsS3BucketName}/${QsS3KeyPrefix}cloudInstallerScripts/ ~/.jfrog_ami/ || cfn_fail - setsebool httpd_can_network_connect 1 -P # CentOS cloned virtual machines do not create a new machine id @@ -381,8 +358,6 @@ Resources: /root/attach_volume.sh || cfn_fail - ansible-galaxy collection install community.general ansible.posix - setsebool httpd_can_network_connect 1 -P aws secretsmanager get-secret-value --secret-id ${ArtifactoryLicensesSecretName} --region ${AWS::Region} | jq -r '{"artifactory_licenses":(.SecretString | fromjson )}' > ~/.jfrog_ami/licenses.json || cfn_fail diff --git a/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml index 4190aa0..366f811 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml @@ -228,31 +228,9 @@ Resources: echo $PATH - # Create virtual env and activate - python3 -m venv ~/venv --system-site-packages + # Activate virtual env source ~/venv/bin/activate - pip install --upgrade pip - pip install wheel - - S3URI=${QsS3Uri} - - yum update --security -y &> /var/log/userdata.yum_security_update.log - - yum install -y git python3 libselinux-python3 - yum install -y postgresql-server postgresql-devel - - # Install Cloudformation helper scripts - pip install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz 2>&1 | tee /var/log/userdata.aws_cfn_bootstrap_install.log - - pip install awscli &> /var/log/userdata.awscli_install.log - - pip install ansible &> /var/log/userdata.ansible_install.log - - mkdir ~/.xray_ami - - aws s3 --region ${AWS::Region} sync s3://${QsS3BucketName}/${QsS3KeyPrefix}cloudInstallerScripts/ ~/.xray_ami/ - setsebool httpd_can_network_connect 1 -P # CentOS cloned virtual machines do not create a new machine id @@ -269,8 +247,6 @@ Resources: lsblk # debug - ansible-galaxy collection install community.general ansible.posix - ansible-playbook /root/.xray_ami/xray.yml --vault-id /root/.vault_pass.txt 2>&1 | tee /var/log/xray.log || cfn_fail rm -rf /root/.secureit.sh From f41fbb3913c0a1de3689034533b35af9e42f1f43 Mon Sep 17 00:00:00 2001 From: Alex Hung Date: Wed, 24 Feb 2021 11:34:12 -0800 Subject: [PATCH 6/8] Update AMI IDs --- .../templates/jfrog-artifactory-ec2-instance.template.yaml | 4 ++-- .../v7153/templates/jfrog-xray-ec2-instance.template.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml index 4e37adc..9b1b777 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml @@ -86,7 +86,7 @@ Mappings: us-west-1: "Artifactory7153": ami-0afdf2d63fc2c0895 us-west-2: - "Artifactory7153": ami-09274621c30cae078 + "Artifactory7153": ami-03c0e58806c907dee ca-central-1: "Artifactory7153": ami-0f422f5980aeba60f eu-central-1: @@ -112,7 +112,7 @@ Mappings: us-gov-east-1: "Artifactory7153": ami-0ec712ae031edcb34 us-gov-west-1: - "Artifactory7153": ami-0b6229d13d677cd16 + "Artifactory7153": ami-08343af72f1764764 ArtifactoryProductMap: JFrog-Container-Registry: "7153": "Jcr7153" diff --git a/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml index 366f811..666f282 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml @@ -64,7 +64,7 @@ Mappings: us-west-1: "3174": ami-006041db0f601e438 us-west-2: - "3174": ami-0d67b78b4f09fa9a7 + "3174": ami-05019f9185d4f809e ca-central-1: "3174": ami-056bafb407aa8e445 eu-central-1: @@ -90,7 +90,7 @@ Mappings: us-gov-east-1: "3174": ami-0349215efccd0d9f6 us-gov-west-1: - "3174": ami-0b52a6d3379d2c20c + "3174": ami-006010831d25af96b Resources: XrayScalingGroup: From 14dcf41a463ef42fc90c3eb60c298332eff3c73b Mon Sep 17 00:00:00 2001 From: Alex Hung Date: Wed, 24 Feb 2021 11:34:27 -0800 Subject: [PATCH 7/8] Other tidying up --- ...artifactory-ec2-existing-vpc.template.yaml | 32 +++++++------------ .../jfrog-xray-ec2-instance.template.yaml | 2 +- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml index 9e6e270..1e646cc 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-existing-vpc.template.yaml @@ -386,7 +386,7 @@ Resources: ArtifactoryCoreInfraStack: Type: AWS::CloudFormation::Stack Properties: - TemplateURL: https://jfrog-marketplace-test-us-east-1.s3.amazonaws.com/marketplace-jfrog-artifactory/templates/jfrog-artifactory-core-infrastructure.template.yaml + TemplateURL: https://jfrog-marketplace-test-us-east-1.s3.amazonaws.com/marketplace-jfrog-artifactory/v7153/templates/jfrog-artifactory-core-infrastructure.template.yaml Parameters: AvailabilityZones: Fn::Join: @@ -500,7 +500,7 @@ Resources: Tags: - Key: Name Value: !Sub ${ArtifactoryProduct}-ec2-instances-sg - GroupDescription: SG for EC2 instances (also permits access using SSH from the bastion host) + GroupDescription: SG for EC2 instances VpcId: !Ref VpcId SecurityGroupIngress: - IpProtocol: tcp @@ -579,21 +579,17 @@ Resources: - "logs:CreateLogStream" - "logs:PutLogEvents" - "logs:DescribeLogStreams" - Resource: "arn:aws:logs:*:*:*" - - Effect: "Allow" - Action: - - "s3:GetObject" - Resource: "*" - - PolicyName: 'SecretsMaanger-policy' + Resource: !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:*:*" + - PolicyName: 'SecretsManager-policy' PolicyDocument: Version: "2012-10-17" Statement: - Effect: "Allow" Action: - "secretsmanager:GetSecretValue" - Resource: "arn:aws:secretsmanager:*:*:secret:*" + Resource: !Sub "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:*" ArtifactoryHostProfile: - Type: 'AWS::IAM::InstanceProfile' + Type: AWS::IAM::InstanceProfile Properties: Roles: - !Ref ArtifactoryHostRole @@ -601,11 +597,11 @@ Resources: ArtifactoryPrimary: Type: AWS::CloudFormation::Stack Properties: - TemplateURL: https://jfrog-marketplace-test-us-east-1.s3.amazonaws.com/marketplace-jfrog-artifactory/templates/jfrog-artifactory-ec2-instance.template.yaml + TemplateURL: https://jfrog-marketplace-test-us-east-1.s3.amazonaws.com/marketplace-jfrog-artifactory/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml Parameters: PrivateSubnetIds: !Join [',', [!Ref PrivateSubnet1Id]] - MinScalingNodes: '1' # Always have 1 MasterNode - MaxScalingNodes: '1' # Always have 1 MasterNode + MinScalingNodes: '1' # Always have 1 Primary Node + MaxScalingNodes: '1' # Always have 1 Primary Node DeploymentTag: !If [IsArtifactory, "ArtifactoryPrimary", "JcrPrimary"] HostRole: !Ref ArtifactoryHostRole ArtifactoryProduct: !Ref ArtifactoryProduct @@ -643,7 +639,7 @@ Resources: DependsOn: ArtifactoryPrimary Type: AWS::CloudFormation::Stack Properties: - TemplateURL: https://jfrog-marketplace-test-us-east-1.s3.amazonaws.com/marketplace-jfrog-artifactory/templates/jfrog-artifactory-ec2-instance.template.yaml + TemplateURL: https://jfrog-marketplace-test-us-east-1.s3.amazonaws.com/marketplace-jfrog-artifactory/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml Parameters: PrivateSubnetIds: !Join [',', [!Ref PrivateSubnet1Id, !Ref PrivateSubnet2Id]] MinScalingNodes: !Ref NumberOfSecondary @@ -725,11 +721,7 @@ Resources: - "logs:CreateLogStream" - "logs:PutLogEvents" - "logs:DescribeLogStreams" - Resource: "arn:aws:logs:*:*:*" - - Effect: "Allow" - Action: - - "s3:GetObject" - Resource: "*" + Resource: !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:*:*" XrayHostProfile: Condition: EnableXray Type: 'AWS::IAM::InstanceProfile' @@ -742,7 +734,7 @@ Resources: DependsOn: ArtifactoryPrimary Type: AWS::CloudFormation::Stack Properties: - TemplateURL: https://jfrog-marketplace-test-us-east-1.s3.amazonaws.com/marketplace-jfrog-artifactory/templates/jfrog-xray-ec2-instance.template.yaml + TemplateURL: https://jfrog-marketplace-test-us-east-1.s3.amazonaws.com/marketplace-jfrog-artifactory/v7153/templates/jfrog-xray-ec2-instance.template.yaml Parameters: PrivateSubnet1Id: !Ref PrivateSubnet1Id PrivateSubnet2Id: !Ref PrivateSubnet2Id diff --git a/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml index 666f282..c10ac91 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml @@ -182,7 +182,7 @@ Resources: IamInstanceProfile: !Ref XrayHostProfile ImageId: !FindInMap - AWSAMIRegionMap - - !Ref 'AWS::Region' + - !Ref AWS::Region - !Ref XrayAmiId SecurityGroups: - !Ref SecurityGroups From 6215731cb74b335f72a787bc373335abb9423b32 Mon Sep 17 00:00:00 2001 From: Alex Hung Date: Wed, 24 Feb 2021 11:42:01 -0800 Subject: [PATCH 8/8] Update AMI IDs for us-east-1 and us-gov-east-1 --- .../templates/jfrog-artifactory-ec2-instance.template.yaml | 4 ++-- .../v7153/templates/jfrog-xray-ec2-instance.template.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml index 9b1b777..20f7ee8 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-artifactory-ec2-instance.template.yaml @@ -80,7 +80,7 @@ Parameters: Mappings: AWSAMIRegionMap: us-east-1: - "Artifactory7153": ami-07a437a0be21eb6c8 + "Artifactory7153": ami-0e25fbf67a59cc92b us-east-2: "Artifactory7153": ami-08f834139a150fb62 us-west-1: @@ -110,7 +110,7 @@ Mappings: sa-east-1: "Artifactory7153": ami-0b831f8403d6979d4 us-gov-east-1: - "Artifactory7153": ami-0ec712ae031edcb34 + "Artifactory7153": ami-0d91ab5cda810462b us-gov-west-1: "Artifactory7153": ami-08343af72f1764764 ArtifactoryProductMap: diff --git a/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml b/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml index c10ac91..3831d34 100644 --- a/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml +++ b/Amazon/Marketplace/v7153/templates/jfrog-xray-ec2-instance.template.yaml @@ -58,7 +58,7 @@ Parameters: Mappings: AWSAMIRegionMap: us-east-1: - "3174": ami-028555fa3b9469dc3 + "3174": ami-0cfe501ba23b450bd us-east-2: "3174": ami-0f4980801d180776a us-west-1: @@ -88,7 +88,7 @@ Mappings: sa-east-1: "3174": ami-0c2acb2f23c3e6743 us-gov-east-1: - "3174": ami-0349215efccd0d9f6 + "3174": ami-06e63f61fa6c13b25 us-gov-west-1: "3174": ami-006010831d25af96b