AWS Glue is a serverless data integration service that makes it easy to discover, prepare, and combine data for analytics, machine learning, and application development
Eg: behaviours
[project]
name = "MYAPP_datasync"
version = "1.0.0-SNAPSHOT.3" # Makefile updates sonar-project.properties
description = "AWS Glue data synchronization script"
readme = "README.md"
requires-python = ">=3.11, <3.12"
sonar.host.url=https://sonarqube.cou.edu/sonar
sonar.projectKey=MYAPP:MYAPP_datasync
sonar.projectName=MYAPP:MYAPP_datasync
sonar.projectVersion=1.0.0-SNAPSHOT.3
sonar.sources=src/main/python
sonar.tests=src/test/python
sonar.language=py
sonar.python.version=3.11
sonar.python.coverage.reportPaths=coverage.xml
sonar.exclusions=**/__pycache__/**, **/*.pyc, .venv/**, .pytest_cache/**, .coverage
sonar.test.inclusions=src/test/python/**/*.py
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
boto3 = "==1.42.58"
pg8000 = "==1.31.5"
[dev-packages]
pytest = "==9.0.2"
pytest-cov = "==7.0.0"
moto = "==5.1.21"
pysonar = "*"
[requires]
python_version = "3.11"
.PHONY: help deploy all venv install test test-cov clean-py
# ========================================================
# WARNING: This file is for development purposes only.
# Do NOT use it for anything else (it can be modified or deleted at any time w/out previous notice).
# You've been warned!
# ========================================================
# ========================================================
# TOOLS REFERENCE, tested w/ Ubuntu 24.04:
# (make) sudo apt install make
# (aws) https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html
# ========================================================
### INI CONFIG
s3_coderepository=myapp-awsinter-dev-s3-coderepository
s3_coderepository_prefix=/python/glue/myapp_datasync/
# Python Environment with Pipenv
# Enforce Python 3.11 for Glue 5.1
export PIPENV_VENV_IN_PROJECT=1
export PIPENV_YES=1
PYTHON_BIN = 3.11
PIPENV = pipenv
# Project Metadata
VERSION = $(shell grep -m 1 "version =" pyproject.toml | cut -d '"' -f 2)
### END CONFIG
help: ## Show this help message
@echo "Project Version: $(VERSION)"
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
clean: sync-version ## Remove .venv, cache and temporary files
rm -rf .venv coverage.xml
find . -type d -name "__pycache__" -exec rm -rf {} +
find . -type f -name "*.pyc" -delete
rm -rf .pytest_cache .coverage
install: sync-version ## Install dependencies using pipenv (auto-installs Python PYTHON_BIN via pyenv if missing)
@if ! pyenv prefix $(PYTHON_BIN) >/dev/null 2>&1; then \
echo "Python $(PYTHON_BIN) not found in pyenv. Installing..."; \
pyenv install -s $(PYTHON_BIN); \
fi
$(PIPENV) --python $(PYTHON_BIN) install --dev
test: sync-version ## Run unit tests with coverage (includes XML report for Sonar)
$(PIPENV) run pytest --cov=src/main/python --cov-report=term-missing --cov-report=xml src/test/python/
sonar: sync-version ## Run SonarQube analysis
$(PIPENV) run pysonar -Dsonar.token=${UOC_SONAR_USER_TOKEN}
sync-version: # Sync version from pyproject.toml to metadata files
@sed -i 's/^sonar\.projectVersion=.*/sonar.projectVersion=$(VERSION)/' sonar-project.properties
s3-remove-dir: # Delete from S3 the folder <app>_datasync
aws s3 rm s3://$(s3_coderepository)$(s3_coderepository_prefix) --recursive
s3-upload-dir: # Upload python directory to s3
aws s3 cp ./src/main/python/ s3://$(s3_coderepository)/python/ --recursive
deploy: sync-version ## Deploy DEV
$(MAKE) s3-remove-dir
$(MAKE) s3-upload-dir
replace-master: ## Replace content of git branch 'master' by that of 'develop'
@if [ -n "$$(git status --porcelain 2>/dev/null)" ]; then \
echo "Error: Pending changes detected. Aborting."; \
git status; \
exit 1; \
fi; \
echo "This will replace 'master' with 'develop' and force push. THIS IS DESTRUCTIVE!"; \
read -p "Are you absolutely sure you want to proceed? (yes/no): " CONFIRMATION; \
if [ "$$CONFIRMATION" != "yes" ]; then \
echo "Operation cancelled."; \
exit 0; \
fi; \
echo "Proceeding..."; \
git checkout master && \
git fetch origin && \
git reset --hard develop && \
git push --force-with-lease origin master && \
git checkout develop
all: ## Run tests and deploy
$(MAKE) test
$(MAKE) deploy