Cartographer DSL

A domain specific language for defining CircleCI pipelines.

Cartographer DSL
- Local Usage
- Running on CircleCI
Examples
Docs: CircleCI

Expand For Full Table Of Contents

Cartographer DSL
Examples
- Minimal
Docs: CircleCI

Why??

YAML is fine at 100 lines, less so at 1,000 lines and almost impossible to maintain at 10,000 lines. Anchors provide some capacity to keep things DRY but they're clunky and don't do enough really.

Enter Cartographer DSL, a domain-specific language based on Jsonnet that gives you the power of a complete programming language.

Some of the benefits:

Config can be more condensed when appropriate (easier to view lots of config all at once)
Loops and variables make it easier to repeat and template steps

I'm also hoping to take some common patterns and turn them into templates/functions/reusable chunks.

I built this because I noticed CircleCI supported dynamic config generation, and I wanted more options than YAML to write my config.

Local Usage

Docker Image

export FILE_NAME=".circleci/_config.jsonnet"
docker run -it \
    -v "$(pwd):/project" thejokersthief/cartographer-dsl \
    cartographer-dsl --file $FILE_NAME

Running on CircleCI

Check out the .circleci folder for a living example.

The below config uses the setup ability for CircleCI pipelines to generate the config dynamically. You'll need to enable this for your project in the project settings under "Advanced", at the bottom of the page.

Then, put your Cartographer file in the .circleci/_config.jsonnet.

Finally, add the following .circleci/config.yml file:

version: 2.1
setup: true
orbs:
  continuation: circleci/[email protected]
jobs:
  generate-config:
    docker:
      - image: thejokersthief/cartographer-dsl:latest
    steps:
      - checkout
      - run:
          name: Generate Pipeline generated_config.yml file
          command: cartographer-dsl --file .circleci/_config.jsonnet > generated_config.yml
      - continuation/continue:
          parameters: '{}'
          configuration_path: generated_config.yml
workflows:
  setup-workflow:
    jobs:
      - generate-config:
          filters:
            branches:
              only:
                - /.*/
            tags:
              only:
                - /.*/

Examples

If you're looking to learn more about writing Jsonnet, there's a very nice tutorial on the official website.

Minimal

local dsl = import '_dsl.libsonnet';

local jobs = dsl.jobs;
local pipeline = dsl.pipeline;
local steps = dsl.steps;
local workflows = dsl.workflows;

pipeline.new(
    workflows=[
        workflows.new(
            'main', jobs=[
                workflows.job(
                    'build',
                    image="ubuntu:20.04",
                    steps=[
                        steps.checkout(),
                        steps.run('echo "Hello World!"', name='Hello, World'),
                    ]
                )
            ]
        ),
    ],
)

Docs: CircleCI

Pipeline

dsl.circleci.pipeline.new

Name	Type	Default
version	string	"2.1",
orbs	Orb	{}
workflows	list[Workflows]	[]
executors	list[Executors]	[]

Example:

local dsl = import '_dsl.libsonnet';

local jobs = dsl.jobs;
local pipeline = dsl.pipeline;
local steps = dsl.steps;
local workflows = dsl.workflows;

pipeline.new(
    workflows=[
        workflows.new(
            'main', jobs=[
                workflows.job(
                    'build',
                    image="ubuntu:20.04",
                    steps=[
                        steps.checkout(),
                        steps.run('echo "Hello World!"', name='Hello, World'),
                    ]
                )
            ]
        ),
    ],
)

Workflows

dsl.circleci.workflows.new

Name	Type	Default
name	string	-
jobs	list[Job]	-

local dsl = import '_dsl.libsonnet';
local workflows = dsl.workflows;

workflows.new(
    "example-workflow",
    jobs=[ ... ]
)

Job

dsl.circleci.workflows.job

Name	Type	Default
name	string	-
requires	list[string]	[]
context	list[string]	[]
filters	Filter	{}
matrix	Matrix	{}
when	string	`null`
steps	list[Step]	[]
image	string	`null`
executor	string	docker
executor_options	Executor	{}
shell	string	`null`
parameters	dict[string]string	{}
working_directory	string	`null`
parallelism	integer	1
environment	dict[string]string	{}
resource_class	string	`null`
circleci_ip_ranges	bool	false

local dsl = import '_dsl.libsonnet';

local steps = dsl.steps;
local workflows = dsl.workflows;

workflows.job(
    'build',
    image="ubuntu:20.04",
    steps=[
        steps.checkout(),
    ]
)

Job Executor Options

dsl.circleci.executors.[docker, machine, macos, windows]

Filter: Branches

dsl.circleci.workflows.filter_branches

Name	Type	Default
only	list[string]	[]
ignore	list[string]	[]

local dsl = import '_dsl.libsonnet';
local workflows = dsl.workflows;

workflows.new(
    "example-workflow",
    jobs=[
        workflows.job(
            name='example-job',
            requires=['another-job'],
            filters=workflows.filters(
                only=['main']
            )
        )
    ]
)

Filter: Tags

dsl.circleci.workflows.filter_tags

Name	Type	Default
only	list[string]	[]
ignore	list[string]	[]

local dsl = import '_dsl.libsonnet';
local workflows = dsl.workflows;

workflows.new(
    "example-workflow",
    jobs=[
        workflows.job(
            name='example-job',
            requires=['another-job'],
            filters=workflows.filters(
                only=['/v1.*/'],
                ignore=['/beta-.*/']
            )
        )
    ]
)

Multiple Filters

local dsl = import '_dsl.libsonnet';
local workflows = dsl.workflows;

workflows.new(
    "example-workflow",
    jobs=[
        workflows.job(
            name='example-job',
            requires=['another-job'],
            filters=(
                workflows.filters(
                    only=['/v1.*/'],
                    ignore=['/beta-.*/']
                )
                + workflows.filters(
                    only=['main']
                )
            )
        )
    ]
)

Matrix

dsl.circleci.workflows.matrix

Name	Type	Default
alias	string	`null`
parameters	dict[string]string	{}
exclude	list[string]	[]

local dsl = import '_dsl.libsonnet';
local workflows = dsl.workflows;

workflows.new(
    "example-workflow",
    jobs=[
        workflows.job(
            name='example-job',
            matrix=workflows.matrix(
                parameters={
                    version: ["0.1", "0.2", "0.3"]
                    platform: ["macos", "windows", "linux"]
                }
            )
        )
    ]
)

Docker

dsl.circleci.executors.docker

Name	Type	Default
image	string	-
name	string	`null`
entrypoint	string	`null`
command	string	`null`
user	string	`null`
environment	dict[string]string	{}
auth	dict[string]string	{}
aws_auth	dict[string]string	{}

local dsl = import '_dsl.libsonnet';
local jobs = dsl.jobs;
local executors = dsl.executors;

jobs.new(
    'build',
    executor="docker",
    executor_options=executors.docker(
        image="ubuntu:20.04",
        auth={'username': 'example', 'password': 'example'}
    ),
    steps=[
        steps.checkout(),
        steps.run('echo "Hello World!"', name='Hello, World'),
    ]
),

Machine

dsl.circleci.executors.machine

Name	Type	Default
image	string	-
docker_layer_caching	bool	false

local dsl = import '_dsl.libsonnet';
local jobs = dsl.jobs;
local executors = dsl.executors;

jobs.new(
    'build',
    executor="machine"
    executor_options=executors.machine(image='ubuntu-2004:202201-02')
    steps=[
        steps.checkout(),
        steps.run('echo "Hello World!"', name='Hello, World'),
    ]
),

MacOS

dsl.circleci.executors.macos

Name	Type	Default
xcode	string	-

local dsl = import '_dsl.libsonnet';
local jobs = dsl.jobs;

jobs.new(
    'build',
    executor="macos"
    executor_options=executors.macos('12.5.1')
    steps=[
        steps.checkout(),
        steps.run('echo "Hello World!"', name='Hello, World'),
    ]
),

Windows

dsl.circleci.executors.windows

The windows executor has no options.

Custom Executor (e.g. from an orb)

local dsl = import '_dsl.libsonnet';
local jobs = dsl.jobs;


// orbs.new({ python: 'circleci/[email protected]' })

jobs.new(
    'build',
    executor="python/default"
    steps=[
        steps.checkout(),
        steps.run('echo "Hello World!"', name='Hello, World'),
    ]
),

Steps

run

dsl.circleci.steps.run

Name	Type	Default
command	string	-
name	string	`null`
shell	string	`null`
environment	dict[string]string	{}
background	bool	false
working_directory	string	"."
no_output_timeout	string	"10m"
when	string	dict

local dsl = import '_dsl.libsonnet';
local steps = dsl.steps;

steps.run('echo "Hello, World!", no_output_timeout="30m)

when

Name	Type	Default
steps	list[Step]	-
when	string	dict

dsl.circleci.steps.when

local dsl = import '_dsl.libsonnet';
local steps = dsl.steps;
local const = dsl.constants;

steps.when(
    condition={
        'equal': ['my-fave-schedule', const.pipeline_values.schedule_name]
    },
    steps=[steps.run('echo "This is my favourite schedule!"')]
)

unless

dsl.circleci.steps.unless

Name	Type	Default
steps	list[Step]	-
when	string	dict

local dsl = import '_dsl.libsonnet';
local steps = dsl.steps;

steps.unless(
    condition={
        'equal': ['my-fave-schedule', const.pipeline_values.schedule_name]
    },
    steps=[steps.run('echo "This is my favourite schedule!"')]
)

checkout

dsl.circleci.steps.checkout

local dsl = import '_dsl.libsonnet';
local steps = dsl.steps;

steps.checkout()

setup_remote_docker

dsl.circleci.steps.setup_remote_docker

Name	Type	Default
version	string	'17.09.0-ce'
docker_layer_caching	bool	false

local dsl = import '_dsl.libsonnet';
local steps = dsl.steps;

steps.setup_remote_docker()

save_cache

dsl.circleci.steps.save_cache

Name	Type	Default
key	string	-
paths	list[string]	-

local dsl = import '_dsl.libsonnet';
local steps = dsl.steps;
local const = dsl.constants;

steps.save_cache(
    key='v1' + const.cache_vars.checksum('testfile.txt'),
    paths=['/cached/path']
)

restore_cache

dsl.circleci.steps.restore_cache

Name	Type	Default
key	string	-

local dsl = import '_dsl.libsonnet';
local steps = dsl.steps;
local const = dsl.constants;

steps.restore_cache('v1' + const.cache_vars.checksum('testfile.txt'))

store_artifacts

dsl.circleci.steps.store_artifacts

Name	Type	Default
path	string	-
destination	string	`null`

local dsl = import '_dsl.libsonnet';
local steps = dsl.steps;

steps.store_artifacts('/my-artifacts-folders/')

store_test_results

dsl.circleci.steps.store_test_results

Name	Type	Default
path	string	-

local dsl = import '_dsl.libsonnet';
local steps = dsl.steps;

steps.store_test_results('/my-artifacts-folders/')

persist_to_workspace

dsl.circleci.steps.persist_to_workspace

Name	Type	Default
root	string	-
paths	list[string]	-

local dsl = import '_dsl.libsonnet';
local steps = dsl.steps;

steps.persist_to_workspace(
    root='/my-project',
    paths=['important-folder']
)

attach_workspace

dsl.circleci.steps.attach_workspace

Name	Type	Default
at	string	-

local dsl = import '_dsl.libsonnet';
local steps = dsl.steps;

steps.attach_workspace('/my-project/important-folder')

add_ssh_keys

dsl.circleci.steps.add_ssh_keys

Name	Type	Default
fingerprints	list[string]	-

local dsl = import '_dsl.libsonnet';
local steps = dsl.steps;

steps.add_ssh_keys()

Executors

dsl.circleci.executors

Name	Type	Default
name	string	-
executor	string	-
executor_options	Executor	-
resource_class	string	`null`
shell	string	`null`
working_directory	string	`null`
environment	dict[string]string	{}

local dsl = import '_dsl.libsonnet';
local pipeline = dsl.pipeline;
local executors = dsl.executors;

pipeline.new(
    executors=[
        // You can define multiple images for the same executor by reusing the name
        executors.new("my-custom-executor", executor=docker, executor_options=executors.docker(image='ubuntu:20.04')),
        executors.new("my-custom-executor", executor=docker, executor_options=executors.docker(image='redis')),

        // Otherwise, they're all unique
        executors.new("my-other-executor", executor=docker, executor_options=executors.docker(image='redis')),
    ]
)

Orbs

local dsl = import '_dsl.libsonnet';
local pipeline = dsl.pipeline;
local orbs = dsl.orbs;

pipeline.new(
    orbs=orbs.new({ python: 'circleci/[email protected]' }),
    jobs=[
        jobs.new(
            'build-and-test',
            executor='python/default',
            steps=[
                steps.checkout(),
                'python/load-cache',
                steps.run('poetry install', name='Install dependencies'),
                'python/save-cache',
                steps.run('poetry run pytest tests', name='Tests'),
            ],
        ),
    ],
)

Name		Name	Last commit message	Last commit date
Latest commit History 52 Commits
.circleci		.circleci
cmd/cartographer-dsl		cmd/cartographer-dsl
docker		docker
dsl		dsl
examples		examples
pkg/cartographer-dsl		pkg/cartographer-dsl
.editorconfig		.editorconfig
.gitignore		.gitignore
Dockerfile		Dockerfile
LICENSE		LICENSE
Makefile		Makefile
README.md		README.md
cartographer-campervan.png		cartographer-campervan.png
config.yaml		config.yaml
errcheck-exclude.txt		errcheck-exclude.txt
go.mod		go.mod
go.sum		go.sum

License

TheJokersThief/CartographerDSL

Folders and files

Latest commit

History

Repository files navigation