Merge pull request #43 from jefferyfry/master

Ansible collection 1.1.0 RT 7.7.8 and Xray 3.8.6
This commit is contained in:
Jeff Fry
2020-09-28 09:15:51 -07:00
committed by GitHub
34 changed files with 341 additions and 97 deletions

11
Ansible/CHANGELOG.md Normal file
View File

@@ -0,0 +1,11 @@
# Changelog
All notable changes to this project will be documented in this file.
## [1.1.0] - 2020-09-27
- Validated for Artifactory 7.7.8 and Xray 3.8.6.
- Added offline support for Artifactory and Xray.
- Added support for configurable Postgres pg_hba.conf.
- Misc fixes due to Artifactory 7.7.8.
- Published 1.1.0 to [Ansible Galaxy](https://galaxy.ansible.com/jfrog/installers).

View File

@@ -12,6 +12,11 @@ This Ansible directory consists of the following directories that support the JF
| collection_version | artifactory_version | xray_version |
|--------------------|---------------------|--------------|
| 1.1.0 | 7.7.8 | 3.8.6 |
| 1.0.9 | 7.7.3 | 3.8.0 |
| 1.0.8 | 7.7.3 | 3.8.0 |
| 1.0.8 | 7.7.1 | 3.5.2 |
| 1.0.8 | 7.6.1 | 3.5.2 |
| 1.0.7 | 7.6.1 | 3.5.2 |
| 1.0.6 | 7.5.0 | 3.3.0 |
| 1.0.6 | 7.4.3 | 3.3.0 |
@@ -84,11 +89,29 @@ ansible_ssh_common_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -A us
eg.
ansible_ssh_common_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -A ubuntu@{{ azureDeployment.deployment.outputs.lbIp.value }} -W %h:%p"'
```
## Upgrades
The Artifactory and Xray roles support software updates. To use a role to perform a software update only, use the _artifactory_upgrade_only_ or _xray_upgrade_only_ variable and specify the version. See the following example.
```
- hosts: artifactory
vars:
artifactory_version: "{{ lookup('env', 'artifactory_version_upgrade') }}"
artifactory_upgrade_only: true
roles:
- artifactory
- hosts: xray
vars:
xray_version: "{{ lookup('env', 'xray_version_upgrade') }}"
xray_upgrade_only: true
roles:
- xray
```
## Building the Collection Archive
1. Go to the [ansible_collections/jfrog/installers directory](ansible_collections/jfrog/installers).
2. Update the galaxy.yml meta file as needed. Update the version.
3. Build the archive.
3. Build the archive. (Requires Ansible 2.9+)
```
ansible-galaxy collection build
```

View File

@@ -9,7 +9,7 @@ namespace: "jfrog"
name: "installers"
# The version of the collection. Must be compatible with semantic versioning
version: "1.0.9"
version: "1.1.0"
# The path to the Markdown (.md) readme file. This path is relative to the root of the collection
readme: "README.md"

View File

@@ -12,7 +12,7 @@ The artifactory role installs the Artifactory Pro software onto the host. Per th
* _db_user_: The database user to configure. eg. "artifactory"
* _db_password_: The database password to configure. "Art1fact0ry"
* _server_name_: This is the server name. eg. "artifactory.54.175.51.178.xip.io"
* _system_file_: Your own [system YAML](https://www.jfrog.com/confluence/display/JFROG/System+YAML+Configuration+File) file can be specified and used. **If specified, this file will be used rather than constructing a file from the parameters above.**
* _artifactory_system_yaml_: Your own [system YAML](https://www.jfrog.com/confluence/display/JFROG/System+YAML+Configuration+File) file can be specified and used. **If specified, this file will be used rather than constructing a file from the parameters above.**
* _binary_store_file_: Your own [binary store file](https://www.jfrog.com/confluence/display/JFROG/Configuring+the+Filestore) can be used. If specified, the default cluster-file-system will not be used.
* _artifactory_upgrade_only_: Perform an software upgrade only. Default is false.
@@ -24,6 +24,8 @@ The artifactory role installs the Artifactory Pro software onto the host. Per th
### secondary vars (vars used by the secondary Artifactory server)
* _artifactory_is_primary_: For the secondary node(s) this must be set to **false**.
Additional variables can be found in [defaults/main.yml](./defaults/main.yml).
## Example Playbook
```
---

View File

@@ -4,7 +4,7 @@
ansible_marketplace: standalone
# The version of Artifactory to install
artifactory_version: 7.7.3
artifactory_version: 7.7.8
# licenses file - specify a licenses file or specify up to 5 licenses
artifactory_license1:
@@ -29,7 +29,7 @@ artifactory_file_store_dir: /data
artifactory_flavour: pro
extra_java_opts: -server -Xms2g -Xmx14g -Xss256k -XX:+UseG1GC
artifactory_system_yaml: system.yaml.j2
artifactory_system_yaml_template: system.yaml.j2
artifactory_tar: https://dl.bintray.com/jfrog/artifactory-pro/org/artifactory/pro/jfrog-artifactory-pro/{{ artifactory_version }}/jfrog-artifactory-pro-{{ artifactory_version }}-linux.tar.gz
artifactory_home: "{{ jfrog_home_directory }}/artifactory"
artifactory_untar_home: "{{ jfrog_home_directory }}/artifactory-{{ artifactory_flavour }}-{{ artifactory_version }}"

View File

@@ -64,6 +64,14 @@
group: "{{ artifactory_group }}"
become: yes
- name: ensure data exists
file:
path: "{{ artifactory_home }}/var/data"
state: directory
owner: "{{ artifactory_user }}"
group: "{{ artifactory_group }}"
become: yes
- name: ensure etc exists
file:
path: "{{ artifactory_home }}/var/etc"
@@ -74,17 +82,17 @@
- name: use specified system yaml
copy:
src: "{{ system_file }}"
dest: "{{ artifactory_home }}/var/etc/system.yaml"
become: yes
when: system_file is defined
- name: configure system yaml
template:
src: "{{ artifactory_system_yaml }}"
dest: "{{ artifactory_home }}/var/etc/system.yaml"
become: yes
when: system_file is not defined
when: artifactory_system_yaml is defined
- name: configure system yaml template
template:
src: "{{ artifactory_system_yaml_template }}"
dest: "{{ artifactory_home }}/var/etc/system.yaml"
become: yes
when: artifactory_system_yaml is not defined
- name: ensure {{ artifactory_home }}/var/etc/security/ exists
file:
@@ -181,7 +189,7 @@
- name: start and enable the primary node
service:
name: artifactory
state: restarted
state: started
become: yes
when: artifactory_is_primary == true
@@ -193,6 +201,6 @@
- name: start and enable the secondary nodes
service:
name: artifactory
state: restarted
state: started
become: yes
when: artifactory_is_primary == false

View File

@@ -1,7 +1,9 @@
---
- name: Nginx Install Block
- name: install nginx
block:
- name: install nginx
- debug:
msg: "Attempting nginx installation without dependencies for potential offline mode."
- name: install nginx without dependencies
package:
name: nginx
state: present
@@ -11,9 +13,11 @@
become: yes
until: package_res is success
rescue:
- name: perform dependency installation
- debug:
msg: "Attempting nginx installation with dependencies for potential online mode."
- name: install dependencies
include_tasks: "{{ ansible_os_family }}.yml"
- name: install nginx
- name: install nginx after dependency installation
package:
name: nginx
state: present

View File

@@ -5,6 +5,17 @@ The postgres role will install Postgresql software and configure a database and
* _db_users_: This is a list of database users to create. eg. db_users: - { db_user: "artifactory", db_password: "Art1fAct0ry" }
* _dbs_: This is the database to create. eg. dbs: - { db_name: "artifactory", db_owner: "artifactory" }
By default, the [_pg_hba.conf_](https://www.postgresql.org/docs/9.1/auth-pg-hba-conf.html) client authentication file is configured for open access for development purposes through the _postgres_allowed_hosts_ variable:
```
postgres_allowed_hosts:
- { type: "host", database: "all", user: "all", address: "0.0.0.0/0", method: "trust"}
```
**THIS SHOULD NOT BE USED FOR PRODUCTION.**
**Update this variable to only allow access from Artifactory and Xray.**
## Example Playbook
```
---

View File

@@ -82,3 +82,8 @@ postgres_server_auto_explain_log_min_duration: -1
# Whether or not to use EXPLAIN ANALYZE.
postgres_server_auto_explain_log_analyze: true
# Sets the hosts that can access the database
postgres_allowed_hosts:
- { type: "host", database: "all", user: "all", address: "0.0.0.0/0", method: "trust"}

View File

@@ -4,12 +4,14 @@
name: python-psycopg2
update_cache: yes
become: yes
ignore_errors: yes
- name: install python3 psycopg2
apt:
name: python3-psycopg2
update_cache: yes
become: yes
ignore_errors: yes
- name: add postgres apt key
apt_key:

View File

@@ -4,4 +4,8 @@ local all all peer
host all all 127.0.0.1/32 md5
host all all ::1/128 md5
## remote connections IPv4
host all all 0.0.0.0/0 trust
{% if postgres_allowed_hosts and postgres_allowed_hosts is iterable %}
{% for host in postgres_allowed_hosts %}
{{ host.type | default('host') }} {{ host.database | default('all') }} {{ host.user | default('all') }} {{ host.address | default('0.0.0.0/0') }} {{ item.auth | default('trust') }}
{% endfor %}
{% endif %}

View File

@@ -11,9 +11,10 @@ The xray role will install Xray software onto the host. An Artifactory server an
* _db_url_: This is the database url. eg. "postgres://10.0.0.59:5432/xraydb?sslmode=disable"
* _db_user_: The database user to configure. eg. "xray"
* _db_password_: The database password to configure. "xray"
* _system_file_: Your own [system YAML](https://www.jfrog.com/confluence/display/JFROG/System+YAML+Configuration+File) file can be specified and used. If specified, this file will be used rather than constructing a file from the parameters above.
* _xray_system_yaml_: Your own [system YAML](https://www.jfrog.com/confluence/display/JFROG/System+YAML+Configuration+File) file can be specified and used. If specified, this file will be used rather than constructing a file from the parameters above.
* _xray_upgrade_only_: Perform an software upgrade only. Default is false.
Additional variables can be found in [defaults/main.yml](./defaults/main.yml).
## Example Playbook
```
---

View File

@@ -4,7 +4,7 @@
ansible_marketplace: standalone
# The version of xray to install
xray_version: 3.5.2
xray_version: 3.8.6
# whether to enable HA
xray_ha_enabled: true
@@ -25,3 +25,5 @@ xray_group: xray
# if this is an upgrade
xray_upgrade_only: false
xray_system_yaml_template: system.yaml.j2

View File

@@ -27,10 +27,16 @@
name: libwxbase3.0-0v5
update_cache: yes
state: present
ignore_errors: yes
become: yes
- name: Install erlang
- name: Install erlang 21.2.1-1
apt:
deb: "{{ xray_home }}/app/third-party/rabbitmq/esl-erlang_21.2.1-1~ubuntu~xenial_amd64.deb"
when: xray_version is version("3.8.0","<")
become: yes
- name: Install erlang 22.3.4.1-1
apt:
deb: "{{ xray_home }}/app/third-party/rabbitmq/esl-erlang_22.3.4.1-1_ubuntu_xenial_amd64.deb"
when: xray_version is version("3.8.0",">=")
become: yes

View File

@@ -11,8 +11,16 @@
state: present
become: yes
- name: Install erlang
- name: Install erlang 21.1.4-1
yum:
name: "{{ xray_home }}/app/third-party/rabbitmq/erlang-21.1.4-1.el7.centos.x86_64.rpm"
state: present
when: xray_version is version("3.8.0","<")
become: yes
- name: Install erlang 22.3.4.1-1
yum:
name: "{{ xray_home }}/app/third-party/rabbitmq/erlang-22.3.4.1-1.el7.centos.x86_64.rpm"
state: present
when: xray_version is version("3.8.0",">=")
become: yes

View File

@@ -52,11 +52,19 @@
group: "{{ xray_group }}"
become: yes
- name: configure system yaml
template:
src: system.yaml.j2
- name: use specified system yaml
copy:
src: "{{ xray_system_yaml }}"
dest: "{{ xray_home }}/var/etc/system.yaml"
become: yes
when: xray_system_yaml is defined
- name: configure system yaml template
template:
src: "{{ xray_system_yaml_template }}"
dest: "{{ xray_home }}/var/etc/system.yaml"
become: yes
when: xray_system_yaml is not defined
- name: ensure {{ xray_home }}/var/etc/security/ exists
file:

View File

@@ -5,7 +5,7 @@ resources:
gitProvider: jefferyfryGithub
path: jefferyfry/JFrog-Cloud-Installers
pipelines:
- name: ansible_aws_azure_automation_pipeline
- name: ansible_automation_pipeline
steps:
- name: execute_aws_ansible_playbook
type: Bash
@@ -53,58 +53,6 @@ pipelines:
- ls
- eval $(ssh-agent -s)
- ssh-add <(echo "$int_ansiblePrivateKey_key")
- ansible-playbook Ansible/test/aws/playbook.yaml
- ansible-playbook Ansible/test/aws/playbook-ha-install.yaml
onComplete:
- echo "AWS Ansible playbook complete."
- name: execute_azure_ansible_playbook
type: Bash
configuration:
runtime:
type: image
image:
auto:
language: java
versions:
- "8"
integrations:
- name: ansibleAzureKeys
- name: ansibleEnvVars
- name: ansiblePrivateKey
inputResources:
- name: ansibleRepo
execution:
onStart:
- echo "Executing Azure Ansible playbook..."
onExecute:
- sudo apt-get update
- sudo apt-get install gnupg2
- sudo apt-get install software-properties-common
- sudo apt-add-repository --yes --update ppa:ansible/ansible
- sudo apt -y --allow-unauthenticated install ansible
- sudo pip install packaging
- sudo pip install msrestazure
- sudo pip install ansible[azure]
- cd dependencyState/resources/ansibleRepo
- echo 'Setting environment variables...'
- export artifactory_version="$int_ansibleEnvVars_artifactory_version"
- export xray_version="$int_ansibleEnvVars_xray_version"
- export artifactory_license1="$int_ansibleEnvVars_artifactory_license1"
- export artifactory_license2="$int_ansibleEnvVars_artifactory_license2"
- export artifactory_license3="$int_ansibleEnvVars_artifactory_license3"
- export master_key="$int_ansibleEnvVars_master_key"
- export join_key="$int_ansibleEnvVars_join_key"
- export ssh_public_key="$int_ansibleEnvVars_ssh_public_key"
- export arm_template="$int_ansibleEnvVars_arm_template"
- export azure_resource_group="$int_ansibleEnvVars_azure_resource_group"
- export clientId="$int_ansibleAzureKeys_appId"
- export clientSecret="$int_ansibleAzureKeys_password"
- export tenantId="$int_ansibleAzureKeys_tenant"
- printenv
- pwd
- ls
- eval $(ssh-agent -s)
- ssh-add <(echo "$int_ansiblePrivateKey_key")
- az login --service-principal -u "$clientId" -p "$clientSecret" --tenant "$tenantId"
- ansible-playbook Ansible/test/azure/playbook.yaml
onComplete:
- echo "Azure Ansible playbook complete."

View File

@@ -84,6 +84,11 @@
- { db_name: "xraydb", db_owner: "xray" }
groups: database
- name: Set up test environment file
copy:
src: ../tests/src/test/resources/testenv_tpl.yaml
dest: ../tests/src/test/resources/testenv.yaml
- name: Set up test environment url
replace:
path: ../tests/src/test/resources/testenv.yaml
@@ -144,8 +149,3 @@
shell:
cmd: ./gradlew clean unified_test
chdir: ../tests/
- name: Cleanup and delete stack
cloudformation:
stack_name: "{{ lookup('env', 'stack_name') }}"
region: "us-east-1"
state: "absent"

View File

@@ -0,0 +1,172 @@
---
- name: Provision AWS test infrastructure
hosts: localhost
tasks:
- shell: 'pwd'
register: cmd
- debug:
msg: "{{ cmd.stdout }}"
- name: Create AWS test system
cloudformation:
stack_name: "{{ lookup('env', 'stack_name') }}"
state: "present"
region: "us-east-1"
disable_rollback: true
template: "{{ lookup('env', 'cfn_template') }}"
template_parameters:
SSHKeyName: "{{ lookup('env', 'ssh_public_key_name') }}"
tags:
Stack: "{{ lookup('env', 'stack_name') }}"
register: AWSDeployment
- name: Get AWS deployment details
debug:
var: AWSDeployment
- name: Add bastion
add_host:
hostname: "{{ AWSDeployment.stack_outputs.BastionInstancePublic }}"
groups: bastion
ansible_user: "ubuntu"
- name: Add new RT primary to host group
add_host:
hostname: "{{ AWSDeployment.stack_outputs.RTPriInstancePrivate }}"
ansible_user: "ubuntu"
ansible_ssh_common_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -A ubuntu@{{ AWSDeployment.stack_outputs.BastionInstancePublic }} -W %h:%p"'
artifactory_version: "{{ lookup('env', 'artifactory_version') }}"
db_url: "jdbc:postgresql://{{ AWSDeployment.stack_outputs.DBInstancePrivate }}:5432/artifactory"
server_name: "{{ AWSDeployment.stack_outputs.ALBHostName }}"
artifactory_is_primary: true
artifactory_license_file: "{{ lookup('env', 'artifactory_license_file') }}"
groups:
- artifactory
- name: Add RT secondaries to host group
add_host:
hostname: "{{ AWSDeployment.stack_outputs.RTSecInstancePrivate }}"
ansible_user: "ubuntu"
ansible_ssh_common_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -A ubuntu@{{ AWSDeployment.stack_outputs.BastionInstancePublic }} -W %h:%p"'
artifactory_version: "{{ lookup('env', 'artifactory_version') }}"
db_url: "jdbc:postgresql://{{ AWSDeployment.stack_outputs.DBInstancePrivate }}:5432/artifactory"
server_name: "{{ AWSDeployment.stack_outputs.ALBHostName }}"
artifactory_is_primary: false
groups:
- artifactory
- name: Add xrays to host group
add_host:
hostname: "{{ AWSDeployment.stack_outputs.XrayInstancePrivate }}"
ansible_user: "ubuntu"
ansible_ssh_common_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -A ubuntu@{{ AWSDeployment.stack_outputs.BastionInstancePublic }} -W %h:%p"'
xray_version: "{{ lookup('env', 'xray_version') }}"
jfrog_url: "http://{{ AWSDeployment.stack_outputs.ALBHostName }}"
master_key: "{{ lookup('env', 'master_key') }}"
join_key: "{{ lookup('env', 'join_key') }}"
db_type: "postgresql"
db_driver: "org.postgresql.Driver"
db_user: "xray"
db_password: "xray"
db_url: "postgres://{{ AWSDeployment.stack_outputs.DBInstancePrivate }}:5432/xraydb?sslmode=disable"
groups: xray
- name: Add DBs to host group
add_host:
hostname: "{{ AWSDeployment.stack_outputs.DBInstancePrivate }}"
ansible_user: "ubuntu"
ansible_ssh_common_args: '-o ProxyCommand="ssh -o StrictHostKeyChecking=no -A ubuntu@{{ AWSDeployment.stack_outputs.BastionInstancePublic }} -W %h:%p"'
db_users:
- { db_user: "artifactory", db_password: "Art1fAct0ry" }
- { db_user: "xray", db_password: "xray" }
dbs:
- { db_name: "artifactory", db_owner: "artifactory" }
- { db_name: "xraydb", db_owner: "xray" }
groups: database
- name: Set up test environment file
copy:
src: ../tests/src/test/resources/testenv_tpl.yaml
dest: ../tests/src/test/resources/testenv.yaml
- name: Set up test environment url
replace:
path: ../tests/src/test/resources/testenv.yaml
regexp: 'urlval'
replace: "http://{{ AWSDeployment.stack_outputs.ALBHostName }}"
- name: Set up test environment external_ip
replace:
path: ../tests/src/test/resources/testenv.yaml
regexp: 'ipval'
replace: "{{ AWSDeployment.stack_outputs.ALBHostName }}"
- name: Set up test environment rt_password
replace:
path: ../tests/src/test/resources/testenv.yaml
regexp: 'passval'
replace: "password"
- name: show testenv.yaml
debug: var=item
with_file:
- ../tests/src/test/resources/testenv.yaml
- name: Wait 300 seconds for port 22
wait_for:
port: 22
host: "{{ AWSDeployment.stack_outputs.BastionInstancePublic }}"
delay: 10
- debug:
msg: "Unified URL is at http://{{ AWSDeployment.stack_outputs.ALBHostName }}"
# apply roles to install software
- hosts: database
roles:
- postgres
- hosts: artifactory
vars:
artifactory_ha_enabled: true
master_key: "{{ lookup('env', 'master_key') }}"
join_key: "{{ lookup('env', 'join_key') }}"
db_download_url: "https://jdbc.postgresql.org/download/postgresql-42.2.12.jar"
db_type: "postgresql"
db_driver: "org.postgresql.Driver"
db_user: "artifactory"
db_password: "Art1fAct0ry"
roles:
- artifactory
- hosts: xray
roles:
- xray
- name: Test
hosts: localhost
tasks:
- name: Run tests
shell:
cmd: ./gradlew clean unified_test
chdir: ../tests/
# Now upgrade
- name: Upgrade
hosts: localhost
tasks:
- pause:
prompt: "Proceed to upgrade?"
minutes: 5
- hosts: artifactory
vars:
artifactory_version: "{{ lookup('env', 'artifactory_version_upgrade') }}"
artifactory_upgrade_only: true
roles:
- artifactory
- hosts: xray
vars:
xray_version: "{{ lookup('env', 'xray_version_upgrade') }}"
xray_upgrade_only: true
roles:
- xray

View File

@@ -1,3 +1,12 @@
#!/usr/bin/env bash
ansible-playbook Ansible/test/aws/playbook.yaml
export stack_name=$1
export cfn_template="~/git/JFrog-Cloud-Installers/Ansible/infra/aws/lb-rt-xray-ha-ubuntu16.json"
export ssh_public_key_name=jeff-ansible
export artifactory_license_file="~/Desktop/artifactory.cluster.license"
export master_key=d8c19a03036f83ea45f2c658e22fdd60
export join_key=d8c19a03036f83ea45f2c658e22fdd61
export ansible_user=ubuntu
export artifactory_version="7.4.3"
export xray_version="3.4.0"
ansible-playbook Ansible/test/aws/playbook-ha-install.yaml

View File

@@ -0,0 +1,14 @@
#!/usr/bin/env bash
export stack_name=$1
export cfn_template="~/git/JFrog-Cloud-Installers/Ansible/infra/aws/lb-rt-xray-ha-ubuntu16.json"
export ssh_public_key_name=jeff-ansible
export artifactory_license_file="~/Desktop/artifactory.cluster.license"
export master_key=d8c19a03036f83ea45f2c658e22fdd60
export join_key=d8c19a03036f83ea45f2c658e22fdd61
export ansible_user=ubuntu
export artifactory_version="7.4.3"
export xray_version="3.4.0"
export artifactory_version_upgrade="7.6.1"
export xray_version_upgrade="3.5.2"
ansible-playbook Ansible/test/aws/playbook-ha-upgrade.yaml

View File

@@ -1,6 +1,6 @@
artifactory:
url: urlval
external_ip: ipval
url: http://Ansib-Appli-1NLZU3V2AGK49-291976964.us-east-1.elb.amazonaws.com
external_ip: Ansib-Appli-1NLZU3V2AGK49-291976964.us-east-1.elb.amazonaws.com
distribution: artifactory_ha
rt_username: admin
rt_password: passval
rt_password: password

View File

@@ -0,0 +1,6 @@
artifactory:
url: urlval
external_ip: ipval
distribution: artifactory_ha
rt_username: admin
rt_password: passval