itwinai

This module provides logic to build containers, run tests with pytest, and more.

Since itwinai is designed for HPC deployment, the containers need to be tested on relevant
computing environments with hardware (e.g., GPUs) and software (e.g. CUDA) not accessible
in standard GitHub actions VMs. Through an in-pipeline deployment of interLink, we can
offload some tests to run on HPC.

By deploying interLink within the CI pipeline, some tests can be offloaded to run on HPC.

Additionally, since HPC systems prefer Singularity/Apptainer images over Docker, this
module enables the conversion and publication of Docker containers as SIF files.

Two CI pipelines are provided: a development pipeline, which is simpler and does not
run tests on HPC, and a release pipeline, where containers undergo thorough testing on
HPC, are converted to Singularity, and are pushed to both Docker and Singularity
container registries.

Installation

dagger install github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54

Entrypoint

Return Type

Itwinai !

Arguments

Name	Type	Default Value	Description
dockerRegistry	String !	"ghcr.io/intertwin-eu"	The Docker registry base URL where the container will be published
singularityRegistry	String !	"registry.egi.eu/dev.intertwin.eu"	Harbor registry namespace (i.e., 'registry/project') where to publish the Singularity images
image	String !	"itwinai-dev"	The name of the container image
nickname	String !	"torch"	A simple name to indicate the flavor of the image. Used to generate the corresponding 'latest' tag
tag	String	null	Tag for the container image. Defaults to random uuid if not provided
container	Container	null	Optional container image to use as itwinai container

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 --docker-registry string --singularity-registry string --image string --nickname string

func (m *MyModule) Example(dockerRegistry string, singularityRegistry string, image string, nickname string) *dagger.Itwinai  {
	return dag.
			Itwinai(dockerRegistry, singularityRegistry, image, nickname)
}

@function
def example(docker_registry: str, singularity_registry: str, image: str, nickname: str, ) -> dagger.Itwinai:
	return (
		dag.itwinai(docker_registry, singularity_registry, image, nickname)
	)

@func()
example(dockerRegistry: string, singularityRegistry: string, image: string, nickname: string, ): Itwinai {
	return dag
		.itwinai(dockerRegistry, singularityRegistry, image, nickname)
}

Types

Singularity 🔗

Manage Singularity images.

client() 🔗

Return base image container with Singularity and Oras clients.

Return Type

Container !

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 singularity --base-image string \
 client

func (m *MyModule) Example(baseImage string) *dagger.Container  {
	return dag.
			Itwinai().
			Singularity(baseImage).
			Client()
}

@function
def example(base_image: str) -> dagger.Container:
	return (
		dag.itwinai()
		.singularity(base_image)
		.client()
	)

@func()
example(baseImage: string): Container {
	return dag
		.itwinai()
		.singularity(baseImage)
		.client()
}

container() 🔗

Export Docker container to a Singularity file and return a container containing the generated SIF.

Return Type

Container !

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 singularity --base-image string \
 container

func (m *MyModule) Example(baseImage string) *dagger.Container  {
	return dag.
			Itwinai().
			Singularity(baseImage).
			Container()
}

@function
def example(base_image: str) -> dagger.Container:
	return (
		dag.itwinai()
		.singularity(base_image)
		.container()
	)

@func()
example(baseImage: string): Container {
	return dag
		.itwinai()
		.singularity(baseImage)
		.container()
}

convert() 🔗

Export Docker container to a Singularity file.

Return Type

File !

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 singularity --base-image string \
 convert

func (m *MyModule) Example(baseImage string) *dagger.File  {
	return dag.
			Itwinai().
			Singularity(baseImage).
			Convert()
}

@function
def example(base_image: str) -> dagger.File:
	return (
		dag.itwinai()
		.singularity(base_image)
		.convert()
	)

@func()
example(baseImage: string): File {
	return dag
		.itwinai()
		.singularity(baseImage)
		.convert()
}

publish() 🔗

Export container and publish it to some registry using oras push (using concurrency).

Return Type

String !

Arguments

Name	Type	Default Value	Description
password	Secret !	-	Password for registry
username	Secret !	-	Username for registry
uri	String !	-	Target URI for the image
concurrency	Integer !	4	Number of parallel threads used during image push

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 singularity --base-image string \
 publish --password env:MYSECRET --username env:MYSECRET --uri string --concurrency integer

func (m *MyModule) Example(ctx context.Context, baseImage string, password *dagger.Secret, username *dagger.Secret, uri string, concurrency int) string  {
	return dag.
			Itwinai().
			Singularity(baseImage).
			Publish(ctx, password, username, uri, concurrency)
}

@function
async def example(base_image: str, password: dagger.Secret, username: dagger.Secret, uri: str, concurrency: int) -> str:
	return await (
		dag.itwinai()
		.singularity(base_image)
		.publish(password, username, uri, concurrency)
	)

@func()
async example(baseImage: string, password: Secret, username: Secret, uri: string, concurrency: number): Promise<string> {
	return dag
		.itwinai()
		.singularity(baseImage)
		.publish(password, username, uri, concurrency)
}

publishSingularity() 🔗

Export container and publish it to some registry using singularity push (slow).

Return Type

String !

Arguments

Name	Type	Default Value	Description
password	Secret !	-	Password for registry
username	Secret !	-	Username for registry
uri	String !	-	Target URI for the image

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 singularity --base-image string \
 publish-singularity --password env:MYSECRET --username env:MYSECRET --uri string

func (m *MyModule) Example(ctx context.Context, baseImage string, password *dagger.Secret, username *dagger.Secret, uri string) string  {
	return dag.
			Itwinai().
			Singularity(baseImage).
			PublishSingularity(ctx, password, username, uri)
}

@function
async def example(base_image: str, password: dagger.Secret, username: dagger.Secret, uri: str) -> str:
	return await (
		dag.itwinai()
		.singularity(base_image)
		.publish_singularity(password, username, uri)
	)

@func()
async example(baseImage: string, password: Secret, username: Secret, uri: string): Promise<string> {
	return dag
		.itwinai()
		.singularity(baseImage)
		.publishSingularity(password, username, uri)
}

remove() 🔗

Remove container from some Harbor registry.

Return Type

String !

Arguments

Name	Type	Default Value	Description
password	Secret !	-	Password for registry
username	Secret !	-	Username for registry
registry	String !	"registry.egi.eu"	The registry URL where the container will be published
project	String !	"dev.intertwin.eu"	The name of the project
name	String !	"itwinai-dev"	The name of the container image
tag	String !	"latest"	Tag for the container image

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 singularity --base-image string \
 remove --password env:MYSECRET --username env:MYSECRET --registry string --project string --name string --tag string

func (m *MyModule) Example(ctx context.Context, baseImage string, password *dagger.Secret, username *dagger.Secret, registry string, project string, name string, tag string) string  {
	return dag.
			Itwinai().
			Singularity(baseImage).
			Remove(ctx, password, username, registry, project, name, tag)
}

@function
async def example(base_image: str, password: dagger.Secret, username: dagger.Secret, registry: str, project: str, name: str, tag: str) -> str:
	return await (
		dag.itwinai()
		.singularity(base_image)
		.remove(password, username, registry, project, name, tag)
	)

@func()
async example(baseImage: string, password: Secret, username: Secret, registry: string, project: string, name: string, tag: string): Promise<string> {
	return dag
		.itwinai()
		.singularity(baseImage)
		.remove(password, username, registry, project, name, tag)
}

InterLink 🔗

Wrapper for interLink service, usefult to communciate with the interLink VK.

client() 🔗

Returns a client for the k3s cluster. If the cluster does not exist, it will create it.

dagger call interlink –values=file:tmp.yaml client teminal

Return Type

Container !

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 interlink --values env:MYSECRET --name string --wait integer \
 client

func (m *MyModule) Example(values *dagger.Secret, name string, wait int) *dagger.Container  {
	return dag.
			Itwinai().
			Interlink(values, name, wait).
			Client()
}

@function
def example(values: dagger.Secret, name: str, wait: int) -> dagger.Container:
	return (
		dag.itwinai()
		.interlink(values, name, wait)
		.client()
	)

@func()
example(values: Secret, name: string, wait: number): Container {
	return dag
		.itwinai()
		.interlink(values, name, wait)
		.client()
}

clusterConfig() 🔗

Returns the config file for the k3s cluster.

Return Type

File !

Arguments

Name	Type	Default Value	Description
local	Boolean !	false	Whether to access the cluster from localhost.

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 interlink --values env:MYSECRET --name string --wait integer \
 cluster-config --local boolean

func (m *MyModule) Example(values *dagger.Secret, name string, wait int, local bool) *dagger.File  {
	return dag.
			Itwinai().
			Interlink(values, name, wait).
			ClusterConfig(local)
}

@function
def example(values: dagger.Secret, name: str, wait: int, local: bool) -> dagger.File:
	return (
		dag.itwinai()
		.interlink(values, name, wait)
		.cluster_config(local)
	)

@func()
example(values: Secret, name: string, wait: number, local: boolean): File {
	return dag
		.itwinai()
		.interlink(values, name, wait)
		.clusterConfig(local)
}

deletePod() 🔗

Delete pod. Returns the result of kubectl delete pod.

Return Type

String !

Arguments

Name	Type	Default Value	Description
name	String !	-	pod name

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 interlink --values env:MYSECRET --name string --wait integer \
 delete-pod --name string

func (m *MyModule) Example(ctx context.Context, values *dagger.Secret, name string, wait int, name1 string) string  {
	return dag.
			Itwinai().
			Interlink(values, name, wait).
			DeletePod(ctx, name1)
}

@function
async def example(values: dagger.Secret, name: str, wait: int, name1: str) -> str:
	return await (
		dag.itwinai()
		.interlink(values, name, wait)
		.delete_pod(name1)
	)

@func()
async example(values: Secret, name: string, wait: number, name1: string): Promise<string> {
	return dag
		.itwinai()
		.interlink(values, name, wait)
		.deletePod(name1)
}

getPodLogs() 🔗

Get pod logs.

Return Type

String !

Arguments

Name	Type	Default Value	Description
name	String !	-	pod name

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 interlink --values env:MYSECRET --name string --wait integer \
 get-pod-logs --name string

func (m *MyModule) Example(ctx context.Context, values *dagger.Secret, name string, wait int, name1 string) string  {
	return dag.
			Itwinai().
			Interlink(values, name, wait).
			GetPodLogs(ctx, name1)
}

@function
async def example(values: dagger.Secret, name: str, wait: int, name1: str) -> str:
	return await (
		dag.itwinai()
		.interlink(values, name, wait)
		.get_pod_logs(name1)
	)

@func()
async example(values: Secret, name: string, wait: number, name1: string): Promise<string> {
	return dag
		.itwinai()
		.interlink(values, name, wait)
		.getPodLogs(name1)
}

getPodStatus() 🔗

Get pod status: Pending, Running, Succeeded, Failed, Unknown. Returns the pod status in that moment.

Return Type

String !

Arguments

Name	Type	Default Value	Description
name	String !	-	pod name

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 interlink --values env:MYSECRET --name string --wait integer \
 get-pod-status --name string

func (m *MyModule) Example(ctx context.Context, values *dagger.Secret, name string, wait int, name1 string) string  {
	return dag.
			Itwinai().
			Interlink(values, name, wait).
			GetPodStatus(ctx, name1)
}

@function
async def example(values: dagger.Secret, name: str, wait: int, name1: str) -> str:
	return await (
		dag.itwinai()
		.interlink(values, name, wait)
		.get_pod_status(name1)
	)

@func()
async example(values: Secret, name: string, wait: number, name1: string): Promise<string> {
	return dag
		.itwinai()
		.interlink(values, name, wait)
		.getPodStatus(name1)
}

interlinkCluster() 🔗

Get interLink VK deployed on a k3s cluster. Returns k3s server as a service.

This is the first method you want to call from this class to setup the k3s cluster and deploy the VK if it does not exist yet. However, if an existing service is available, it will reuse it.

Return Type

Service !

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 interlink --values env:MYSECRET --name string --wait integer \
 interlink-cluster

func (m *MyModule) Example(values *dagger.Secret, name string, wait int) *dagger.Service  {
	return dag.
			Itwinai().
			Interlink(values, name, wait).
			InterlinkCluster()
}

@function
def example(values: dagger.Secret, name: str, wait: int) -> dagger.Service:
	return (
		dag.itwinai()
		.interlink(values, name, wait)
		.interlink_cluster()
	)

@func()
example(values: Secret, name: string, wait: number): Service {
	return dag
		.itwinai()
		.interlink(values, name, wait)
		.interlinkCluster()
}

status() 🔗

Get k8s cluster status, including nodes and pods under default namespace.

Return Type

String !

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 interlink --values env:MYSECRET --name string --wait integer \
 status

func (m *MyModule) Example(ctx context.Context, values *dagger.Secret, name string, wait int) string  {
	return dag.
			Itwinai().
			Interlink(values, name, wait).
			Status(ctx)
}

@function
async def example(values: dagger.Secret, name: str, wait: int) -> str:
	return await (
		dag.itwinai()
		.interlink(values, name, wait)
		.status()
	)

@func()
async example(values: Secret, name: string, wait: number): Promise<string> {
	return dag
		.itwinai()
		.interlink(values, name, wait)
		.status()
}

submitPod() 🔗

Submit pod to k8s cluster. Returns the result of submission.

Return Type

String !

Arguments

Name	Type	Default Value	Description
manifest	String !	-	pod manifest serialized as string.

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 interlink --values env:MYSECRET --name string --wait integer \
 submit-pod --manifest string

func (m *MyModule) Example(ctx context.Context, values *dagger.Secret, name string, wait int, manifest string) string  {
	return dag.
			Itwinai().
			Interlink(values, name, wait).
			SubmitPod(ctx, manifest)
}

@function
async def example(values: dagger.Secret, name: str, wait: int, manifest: str) -> str:
	return await (
		dag.itwinai()
		.interlink(values, name, wait)
		.submit_pod(manifest)
	)

@func()
async example(values: Secret, name: string, wait: number, manifest: string): Promise<string> {
	return dag
		.itwinai()
		.interlink(values, name, wait)
		.submitPod(manifest)
}

teardown() 🔗

Stop the k3s service on which interLink VK is running.

Returns: dagger.Service: k3s service.

Return Type

Void !

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 interlink --values env:MYSECRET --name string --wait integer \
 teardown

func (m *MyModule) Example(ctx context.Context, values *dagger.Secret, name string, wait int)   {
	return dag.
			Itwinai().
			Interlink(values, name, wait).
			Teardown(ctx)
}

@function
async def example(values: dagger.Secret, name: str, wait: int) -> None:
	return await (
		dag.itwinai()
		.interlink(values, name, wait)
		.teardown()
	)

@func()
async example(values: Secret, name: string, wait: number): Promise<void> {
	return dag
		.itwinai()
		.interlink(values, name, wait)
		.teardown()
}

testOffloading() 🔗

Test container offloading mechanism on remote HPC using interLink by running simple tests.

Return Type

String !

Arguments

Name	Type	Default Value	Description
partition	String !	"dev"	HPC partition on which to test the offloading

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 interlink --values env:MYSECRET --name string --wait integer \
 test-offloading --partition string

func (m *MyModule) Example(ctx context.Context, values *dagger.Secret, name string, wait int, partition string) string  {
	return dag.
			Itwinai().
			Interlink(values, name, wait).
			TestOffloading(ctx, partition)
}

@function
async def example(values: dagger.Secret, name: str, wait: int, partition: str) -> str:
	return await (
		dag.itwinai()
		.interlink(values, name, wait)
		.test_offloading(partition)
	)

@func()
async example(values: Secret, name: string, wait: number, partition: string): Promise<string> {
	return dag
		.itwinai()
		.interlink(values, name, wait)
		.testOffloading(partition)
}

waitPod() 🔗

Wait for pod termination (Succeeded, Failed) or timeout. Returns last pod status and logs detected.

Return Type

[String ! ] !

Arguments

Name	Type	Default Value	Description
name	String !	-	pod name
timeout	Integer !	300	timeout in seconds
pollInterval	Integer !	5	how often to check the pod status

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 interlink --values env:MYSECRET --name string --wait integer \
 wait-pod --name string --timeout integer --poll-interval integer

func (m *MyModule) Example(ctx context.Context, values *dagger.Secret, name string, wait int, name1 string, timeout int, pollInterval int) []string  {
	return dag.
			Itwinai().
			Interlink(values, name, wait).
			WaitPod(ctx, name1, timeout, pollInterval)
}

@function
async def example(values: dagger.Secret, name: str, wait: int, name1: str, timeout: int, poll_interval: int) -> List[str]:
	return await (
		dag.itwinai()
		.interlink(values, name, wait)
		.wait_pod(name1, timeout, poll_interval)
	)

@func()
async example(values: Secret, name: string, wait: number, name1: string, timeout: number, pollInterval: number): Promise<string[]> {
	return dag
		.itwinai()
		.interlink(values, name, wait)
		.waitPod(name1, timeout, pollInterval)
}

Itwinai 🔗

CI/CD for container images of itwinai.

container() 🔗

Return Type

Container

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 --docker-registry string --singularity-registry string --image string --nickname string container

func (m *MyModule) Example(dockerRegistry string, singularityRegistry string, image string, nickname string) *dagger.Container  {
	return dag.
			Itwinai(dockerRegistry, singularityRegistry, image, nickname).
			Container()
}

@function
def example(docker_registry: str, singularity_registry: str, image: str, nickname: str, ) -> dagger.Container:
	return (
		dag.itwinai(docker_registry, singularity_registry, image, nickname)
		.container()
	)

@func()
example(dockerRegistry: string, singularityRegistry: string, image: string, nickname: string, ): Container {
	return dag
		.itwinai(dockerRegistry, singularityRegistry, image, nickname)
		.container()
}

buildContainer() 🔗

Build itwinai container image from existing Dockerfile

Return Type

Itwinai !

Arguments

Name	Type	Default Value	Description
context	Directory !	-	location of source directory
dockerfile	String !	-	location of Dockerfile
buildArgs	[String ! ]	null	Comma-separated build args
buildArm	Boolean !	false	Whether to build for ARM

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 --docker-registry string --singularity-registry string --image string --nickname string build-container --context DIR_PATH --dockerfile string --build-arm boolean

func (m *MyModule) Example(dockerRegistry string, singularityRegistry string, image string, nickname string, context *dagger.Directory, dockerfile string, buildArm bool) *dagger.Itwinai  {
	return dag.
			Itwinai(dockerRegistry, singularityRegistry, image, nickname).
			BuildContainer(context, dockerfile, buildArm)
}

@function
def example(docker_registry: str, singularity_registry: str, image: str, nickname: str, context: dagger.Directory, dockerfile: str, build_arm: bool) -> dagger.Itwinai:
	return (
		dag.itwinai(docker_registry, singularity_registry, image, nickname)
		.build_container(context, dockerfile, build_arm)
	)

@func()
example(dockerRegistry: string, singularityRegistry: string, image: string, nickname: string, context: Directory, dockerfile: string, buildArm: boolean): Itwinai {
	return dag
		.itwinai(dockerRegistry, singularityRegistry, image, nickname)
		.buildContainer(context, dockerfile, buildArm)
}

devPipeline() 🔗

CI pipeline for pre-release containers. Tests are only local.

Return Type

String !

Arguments

Name	Type	Default Value	Description
tagTemplate	String !	"${itwinai_version}-torch${framework_version}-${os_version}"	Custom image tag pattern.
framework	Enum !	"TORCH"	ML framework in container
skipSingularity	Boolean !	false	Avoid publishing a Singularity image
password	Secret	null	Password for Singularity registry
username	Secret	null	Username for Singularity registry

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 --docker-registry string --singularity-registry string --image string --nickname string dev-pipeline --tag-template string --skip-singularity boolean

func (m *MyModule) Example(ctx context.Context, dockerRegistry string, singularityRegistry string, image string, nickname string, tagTemplate string, framework , skipSingularity bool) string  {
	return dag.
			Itwinai(dockerRegistry, singularityRegistry, image, nickname).
			DevPipeline(ctx, tagTemplate, framework, skipSingularity)
}

@function
async def example(docker_registry: str, singularity_registry: str, image: str, nickname: str, tag_template: str, framework: , skip_singularity: bool) -> str:
	return await (
		dag.itwinai(docker_registry, singularity_registry, image, nickname)
		.dev_pipeline(tag_template, framework, skip_singularity)
	)

@func()
async example(dockerRegistry: string, singularityRegistry: string, image: string, nickname: string, tagTemplate: string, framework: , skipSingularity: boolean): Promise<string> {
	return dag
		.itwinai(dockerRegistry, singularityRegistry, image, nickname)
		.devPipeline(tagTemplate, framework, skipSingularity)
}

interlink() 🔗

Get interLink service.

Return Type

InterLink !

Arguments

Name	Type	Default Value	Description
values	Secret !	-	interLink installer configuration
name	String !	"interlink-cluster"	Name of the k3s cluster in which the interLink VK is deployed
wait	Integer !	60	Sleep time (seconds) needed to wait for the VK to appear in the k3s cluster.
vkName	String	null	Name of the interLink VK. Automatically detected from values if not given
kubernetes	Service	null	Endpoint to exsisting k3s service to attach to. Example (from the CLI): tcp://localhost:1997

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 --docker-registry string --singularity-registry string --image string --nickname string interlink --values env:MYSECRET --name string --wait integer

func (m *MyModule) Example(dockerRegistry string, singularityRegistry string, image string, nickname string, values *dagger.Secret, name string, wait int) *dagger.ItwinaiInterLink  {
	return dag.
			Itwinai(dockerRegistry, singularityRegistry, image, nickname).
			Interlink(values, name, wait)
}

@function
def example(docker_registry: str, singularity_registry: str, image: str, nickname: str, values: dagger.Secret, name: str, wait: int) -> dagger.ItwinaiInterLink:
	return (
		dag.itwinai(docker_registry, singularity_registry, image, nickname)
		.interlink(values, name, wait)
	)

@func()
example(dockerRegistry: string, singularityRegistry: string, image: string, nickname: string, values: Secret, name: string, wait: number): ItwinaiInterLink {
	return dag
		.itwinai(dockerRegistry, singularityRegistry, image, nickname)
		.interlink(values, name, wait)
}

logs() 🔗

Print the logs generted so far. Useful to terminate a chain of steps returning Self, preventing lazy execution of it.

Return Type

String !

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 --docker-registry string --singularity-registry string --image string --nickname string logs

func (m *MyModule) Example(ctx context.Context, dockerRegistry string, singularityRegistry string, image string, nickname string) string  {
	return dag.
			Itwinai(dockerRegistry, singularityRegistry, image, nickname).
			Logs(ctx)
}

@function
async def example(docker_registry: str, singularity_registry: str, image: str, nickname: str, ) -> str:
	return await (
		dag.itwinai(docker_registry, singularity_registry, image, nickname)
		.logs()
	)

@func()
async example(dockerRegistry: string, singularityRegistry: string, image: string, nickname: string, ): Promise<string> {
	return dag
		.itwinai(dockerRegistry, singularityRegistry, image, nickname)
		.logs()
}

publish() 🔗

Push container to registry. Multi-arch support.

Return Type

Itwinai !

Arguments

Name	Type	Default Value	Description
uri	String	null	Optional target URI for the image

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 --docker-registry string --singularity-registry string --image string --nickname string publish

func (m *MyModule) Example(dockerRegistry string, singularityRegistry string, image string, nickname string) *dagger.Itwinai  {
	return dag.
			Itwinai(dockerRegistry, singularityRegistry, image, nickname).
			Publish()
}

@function
def example(docker_registry: str, singularity_registry: str, image: str, nickname: str, ) -> dagger.Itwinai:
	return (
		dag.itwinai(docker_registry, singularity_registry, image, nickname)
		.publish()
	)

@func()
example(dockerRegistry: string, singularityRegistry: string, image: string, nickname: string, ): Itwinai {
	return dag
		.itwinai(dockerRegistry, singularityRegistry, image, nickname)
		.publish()
}

publishSingularity() 🔗

Convert itwinai container to Singularity and push it to some registry.

Return Type

Itwinai !

Arguments

Name	Type	Default Value	Description
password	Secret !	-	Password for Singularity registry
username	Secret !	-	Username for Singularity registry
uri	String	null	Optional target URI for the image

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 --docker-registry string --singularity-registry string --image string --nickname string publish-singularity --password env:MYSECRET --username env:MYSECRET

func (m *MyModule) Example(dockerRegistry string, singularityRegistry string, image string, nickname string, password *dagger.Secret, username *dagger.Secret) *dagger.Itwinai  {
	return dag.
			Itwinai(dockerRegistry, singularityRegistry, image, nickname).
			PublishSingularity(password, username)
}

@function
def example(docker_registry: str, singularity_registry: str, image: str, nickname: str, password: dagger.Secret, username: dagger.Secret) -> dagger.Itwinai:
	return (
		dag.itwinai(docker_registry, singularity_registry, image, nickname)
		.publish_singularity(password, username)
	)

@func()
example(dockerRegistry: string, singularityRegistry: string, image: string, nickname: string, password: Secret, username: Secret): Itwinai {
	return dag
		.itwinai(dockerRegistry, singularityRegistry, image, nickname)
		.publishSingularity(password, username)
}

releasePipeline() 🔗

CI pipeline for release containers. Test on HPC and generate Singularity images.

Return Type

String !

Arguments

Name	Type	Default Value	Description
values	Secret !	-	interLink installer configuration
tagTemplate	String	null	Custom image tag pattern. Example: ‘${itwinai_version}-torch${framework_version}-${os_version}’
framework	Enum !	"TORCH"	ML framework in container
skipHpc	Boolean !	false	Skip tests on remote HPC
skipSingularity	Boolean !	false	Avoid publishing a Singularity image
kubernetes	Service	null	Endpoint to exsisting k3s service to attach to. Example (from the CLI): tcp://localhost:1997
interlinkClusterName	String !	"interlink-cluster"	Name of the k3s cluster in which the interLink VK is deployed
password	Secret	null	Password for Singularity registry
username	Secret	null	Username for Singularity registry

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 --docker-registry string --singularity-registry string --image string --nickname string release-pipeline --values env:MYSECRET --skip-hpc boolean --skip-singularity boolean --interlink-cluster-name string

func (m *MyModule) Example(ctx context.Context, dockerRegistry string, singularityRegistry string, image string, nickname string, values *dagger.Secret, framework , skipHpc bool, skipSingularity bool, interlinkClusterName string) string  {
	return dag.
			Itwinai(dockerRegistry, singularityRegistry, image, nickname).
			ReleasePipeline(ctx, values, framework, skipHpc, skipSingularity, interlinkClusterName)
}

@function
async def example(docker_registry: str, singularity_registry: str, image: str, nickname: str, values: dagger.Secret, framework: , skip_hpc: bool, skip_singularity: bool, interlink_cluster_name: str) -> str:
	return await (
		dag.itwinai(docker_registry, singularity_registry, image, nickname)
		.release_pipeline(values, framework, skip_hpc, skip_singularity, interlink_cluster_name)
	)

@func()
async example(dockerRegistry: string, singularityRegistry: string, image: string, nickname: string, values: Secret, framework: , skipHpc: boolean, skipSingularity: boolean, interlinkClusterName: string): Promise<string> {
	return dag
		.itwinai(dockerRegistry, singularityRegistry, image, nickname)
		.releasePipeline(values, framework, skipHpc, skipSingularity, interlinkClusterName)
}

singularity() 🔗

Access Singularity module.

Return Type

Singularity !

Arguments

Name	Type	Default Value	Description
baseImage	String !	"quay.io/singularity/docker2singularity"	Base Singularity image
docker	Container	null	Docker container to convert to Singularity. If given it overrides the current itwinai container.

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 --docker-registry string --singularity-registry string --image string --nickname string singularity --base-image string

func (m *MyModule) Example(dockerRegistry string, singularityRegistry string, image string, nickname string, baseImage string) *dagger.ItwinaiSingularity  {
	return dag.
			Itwinai(dockerRegistry, singularityRegistry, image, nickname).
			Singularity(baseImage)
}

@function
def example(docker_registry: str, singularity_registry: str, image: str, nickname: str, base_image: str) -> dagger.ItwinaiSingularity:
	return (
		dag.itwinai(docker_registry, singularity_registry, image, nickname)
		.singularity(base_image)
	)

@func()
example(dockerRegistry: string, singularityRegistry: string, image: string, nickname: string, baseImage: string): ItwinaiSingularity {
	return dag
		.itwinai(dockerRegistry, singularityRegistry, image, nickname)
		.singularity(baseImage)
}

testHpc() 🔗

Test container on remote HPC using interLink. Note that the container image must already exist in some publicly accessible containers registry.

Return Type

Itwinai !

Arguments

Name	Type	Default Value	Description
values	Secret !	-	interLink installer configuration
name	String !	"interlink-cluster"	Name of the k3s cluster in which the interLink VK is deployed
wait	Integer !	60	Sleep time (seconds) needed to wait for the VK to appear in the k3s cluster.
kubernetes	Service	null	Endpoint to exsisting k3s service to attach to. Example (from the CLI): tcp://localhost:1997
image	String	null	Container image to test on HPC. If given, it will override any itwinai container previously created by other functions in the chain. Example: docker://ghcr.io/intertwin-eu/itwinai:latest

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 --docker-registry string --singularity-registry string --image string --nickname string test-hpc --values env:MYSECRET --name string --wait integer

func (m *MyModule) Example(dockerRegistry string, singularityRegistry string, image string, nickname string, values *dagger.Secret, name string, wait int) *dagger.Itwinai  {
	return dag.
			Itwinai(dockerRegistry, singularityRegistry, image, nickname).
			TestHpc(values, name, wait)
}

@function
def example(docker_registry: str, singularity_registry: str, image: str, nickname: str, values: dagger.Secret, name: str, wait: int) -> dagger.Itwinai:
	return (
		dag.itwinai(docker_registry, singularity_registry, image, nickname)
		.test_hpc(values, name, wait)
	)

@func()
example(dockerRegistry: string, singularityRegistry: string, image: string, nickname: string, values: Secret, name: string, wait: number): Itwinai {
	return dag
		.itwinai(dockerRegistry, singularityRegistry, image, nickname)
		.testHpc(values, name, wait)
}

testLocal() 🔗

Test itwinai container image with pytest on non-HPC environments.

Return Type

Itwinai !

Arguments

Name	Type	Default Value	Description
cmd	[String ! ]	null	Command to run tests

Example

dagger -m github.com/interTwin-eu/itwinai/ci@66e851b31f2ac9aa3c62cad260a4c40bd1730c54 call \
 --docker-registry string --singularity-registry string --image string --nickname string test-local

func (m *MyModule) Example(dockerRegistry string, singularityRegistry string, image string, nickname string) *dagger.Itwinai  {
	return dag.
			Itwinai(dockerRegistry, singularityRegistry, image, nickname).
			TestLocal()
}

@function
def example(docker_registry: str, singularity_registry: str, image: str, nickname: str, ) -> dagger.Itwinai:
	return (
		dag.itwinai(docker_registry, singularity_registry, image, nickname)
		.test_local()
	)

@func()
example(dockerRegistry: string, singularityRegistry: string, image: string, nickname: string, ): Itwinai {
	return dag
		.itwinai(dockerRegistry, singularityRegistry, image, nickname)
		.testLocal()
}