From eea9b9a4d0039a642b030c864694f3c00016ece3 Mon Sep 17 00:00:00 2001 From: Ashwin Ramaswami Date: Wed, 23 Aug 2023 18:31:19 +0000 Subject: [PATCH 1/7] Fix kubernetes cert connections --- codalab/worker/main.py | 7 +++---- codalab/worker_manager/kubernetes_worker_manager.py | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/codalab/worker/main.py b/codalab/worker/main.py index ef98f0c9e..4b0a01774 100644 --- a/codalab/worker/main.py +++ b/codalab/worker/main.py @@ -4,6 +4,7 @@ # tutorial on the CodaLab documentation. import argparse +import base64 import getpass import os import logging @@ -326,11 +327,9 @@ def main(): # Create temp file to store kubernetes cert, as we need to pass in a file path. # TODO: Delete the file afterwards (upon CodaLab service stop?) with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: - f.write( - args.kubernetes_cert.replace(r'\n', '\n') - ) # Properly add newlines, which appear as "\n" if specified in the environment variable. + f.write(base64.b64decode(args.cert).decode()) kubernetes_cert_path = f.name - logger.info('Temporarily writing kubernetes cert to: %s', kubernetes_cert_path) + logger.info('Temporarily writing kubernetes cert to: %s', kubernetes_cert_path) else: kubernetes_cert_path = args.kubernetes_cert_path bundle_runtime_class = KubernetesRuntime( diff --git a/codalab/worker_manager/kubernetes_worker_manager.py b/codalab/worker_manager/kubernetes_worker_manager.py index 254faf64a..529534cab 100644 --- a/codalab/worker_manager/kubernetes_worker_manager.py +++ b/codalab/worker_manager/kubernetes_worker_manager.py @@ -7,6 +7,7 @@ 'Please run: pip install kubernetes' ) +import base64 import logging import os import uuid @@ -94,11 +95,9 @@ def __init__(self, args): # Create temp file to store kubernetes cert, as we need to pass in a file path. # TODO: Delete the file afterwards (upon CodaLab service stop?) with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: - f.write( - args.cert.replace(r'\n', '\n') - ) # Properly add newlines, which appear as "\n" if specified in the environment variable. + f.write(base64.b64decode(args.cert).decode()) cert_path = f.name - logger.info('Temporarily writing kubernetes cert to: %s', cert_path) + logger.info('Temporarily writing kubernetes cert to: %s', cert_path) else: cert_path = args.cert_path configuration.ssl_ca_cert = cert_path From 278b6296a472e06060226913b9aed6de4e807d7b Mon Sep 17 00:00:00 2001 From: Ashwin Ramaswami Date: Wed, 23 Aug 2023 18:34:53 +0000 Subject: [PATCH 2/7] Update arg descriptions --- codalab/worker/main.py | 2 +- codalab/worker_manager/kubernetes_worker_manager.py | 2 +- codalab_service.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/codalab/worker/main.py b/codalab/worker/main.py index 4b0a01774..da6cafc76 100644 --- a/codalab/worker/main.py +++ b/codalab/worker/main.py @@ -222,7 +222,7 @@ def parse_args(): parser.add_argument( '--kubernetes-cert', type=str, - help='Contents of the SSL cert for the Kubernetes cluster. Only applicable if --bundle-runtime is set to kubernetes and --kubernetes-cert-path is set to /dev/null.', + help='Base64 encoded contents of the SSL cert for the Kubernetes cluster. Only applicable if --bundle-runtime is set to kubernetes and --kubernetes-cert-path is set to /dev/null.', ) return parser.parse_args() diff --git a/codalab/worker_manager/kubernetes_worker_manager.py b/codalab/worker_manager/kubernetes_worker_manager.py index 529534cab..75febee07 100644 --- a/codalab/worker_manager/kubernetes_worker_manager.py +++ b/codalab/worker_manager/kubernetes_worker_manager.py @@ -52,7 +52,7 @@ def add_arguments_to_subparser(subparser: ArgumentParser) -> None: subparser.add_argument( '--cert', type=str, - help='Contents of the SSL cert for the Kubernetes cluster', + help='Base64 encoded contents of the SSL cert for the Kubernetes cluster', required=True, ) subparser.add_argument( diff --git a/codalab_service.py b/codalab_service.py index 451f69257..e5c94f35f 100755 --- a/codalab_service.py +++ b/codalab_service.py @@ -530,7 +530,7 @@ def has_callable_default(self): CodalabArg( name=f'worker_manager_{worker_manager_type}_kubernetes_cert', type=str, - help='Contents of the generated SSL cert for the Kubernetes worker manager', + help='Base64 encoded contents of the generated SSL cert for the Kubernetes worker manager', ), ] From a63d290f0a5b28d07b1c111205d35f28df221fe8 Mon Sep 17 00:00:00 2001 From: Ashwin Ramaswami Date: Wed, 6 Sep 2023 21:36:28 +0000 Subject: [PATCH 3/7] Use quotes, don't use base 64 encoding for cert value --- codalab/worker/main.py | 5 ++--- codalab/worker_manager/kubernetes_worker_manager.py | 5 ++--- codalab_service.py | 2 +- docker_config/compose_files/docker-compose.yml | 4 ++-- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/codalab/worker/main.py b/codalab/worker/main.py index da6cafc76..c2ea1790a 100644 --- a/codalab/worker/main.py +++ b/codalab/worker/main.py @@ -4,7 +4,6 @@ # tutorial on the CodaLab documentation. import argparse -import base64 import getpass import os import logging @@ -222,7 +221,7 @@ def parse_args(): parser.add_argument( '--kubernetes-cert', type=str, - help='Base64 encoded contents of the SSL cert for the Kubernetes cluster. Only applicable if --bundle-runtime is set to kubernetes and --kubernetes-cert-path is set to /dev/null.', + help='Contents of the SSL cert for the Kubernetes cluster. Only applicable if --bundle-runtime is set to kubernetes and --kubernetes-cert-path is set to /dev/null.', ) return parser.parse_args() @@ -327,7 +326,7 @@ def main(): # Create temp file to store kubernetes cert, as we need to pass in a file path. # TODO: Delete the file afterwards (upon CodaLab service stop?) with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: - f.write(base64.b64decode(args.cert).decode()) + f.write(args.cert.replace(r'\n', '\n')) kubernetes_cert_path = f.name logger.info('Temporarily writing kubernetes cert to: %s', kubernetes_cert_path) else: diff --git a/codalab/worker_manager/kubernetes_worker_manager.py b/codalab/worker_manager/kubernetes_worker_manager.py index 75febee07..dc96bc1b4 100644 --- a/codalab/worker_manager/kubernetes_worker_manager.py +++ b/codalab/worker_manager/kubernetes_worker_manager.py @@ -7,7 +7,6 @@ 'Please run: pip install kubernetes' ) -import base64 import logging import os import uuid @@ -52,7 +51,7 @@ def add_arguments_to_subparser(subparser: ArgumentParser) -> None: subparser.add_argument( '--cert', type=str, - help='Base64 encoded contents of the SSL cert for the Kubernetes cluster', + help='Contents of the SSL cert for the Kubernetes cluster', required=True, ) subparser.add_argument( @@ -95,7 +94,7 @@ def __init__(self, args): # Create temp file to store kubernetes cert, as we need to pass in a file path. # TODO: Delete the file afterwards (upon CodaLab service stop?) with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: - f.write(base64.b64decode(args.cert).decode()) + f.write(args.cert.replace(r'\n', '\n')) cert_path = f.name logger.info('Temporarily writing kubernetes cert to: %s', cert_path) else: diff --git a/codalab_service.py b/codalab_service.py index e5c94f35f..451f69257 100755 --- a/codalab_service.py +++ b/codalab_service.py @@ -530,7 +530,7 @@ def has_callable_default(self): CodalabArg( name=f'worker_manager_{worker_manager_type}_kubernetes_cert', type=str, - help='Base64 encoded contents of the generated SSL cert for the Kubernetes worker manager', + help='Contents of the generated SSL cert for the Kubernetes worker manager', ), ] diff --git a/docker_config/compose_files/docker-compose.yml b/docker_config/compose_files/docker-compose.yml index 4012a0d06..5f266fb2f 100644 --- a/docker_config/compose_files/docker-compose.yml +++ b/docker_config/compose_files/docker-compose.yml @@ -303,7 +303,7 @@ services: --cluster-host ${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CLUSTER_HOST} --auth-token ${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_AUTH_TOKEN} --cert-path ${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CERT_PATH} - --cert ${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CERT} + --cert "${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CERT}" --cpus ${CODALAB_WORKER_MANAGER_CPU_DEFAULT_CPUS} --memory-mb ${CODALAB_WORKER_MANAGER_CPU_DEFAULT_MEMORY_MB} <<: *codalab-base @@ -337,7 +337,7 @@ services: --cluster-host ${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CLUSTER_HOST} --auth-token ${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_AUTH_TOKEN} --cert-path ${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT_PATH} - --cert ${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT} + --cert "${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT}" --cpus ${CODALAB_WORKER_MANAGER_GPU_DEFAULT_CPUS} --gpus ${CODALAB_WORKER_MANAGER_DEFAULT_GPUS} --memory-mb ${CODALAB_WORKER_MANAGER_GPU_DEFAULT_MEMORY_MB} From f59b1db2722686dd939e58c9b5e13581ebb24d5b Mon Sep 17 00:00:00 2001 From: Ashwin Ramaswami Date: Wed, 6 Sep 2023 22:36:51 +0000 Subject: [PATCH 4/7] fix --- codalab/worker/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codalab/worker/main.py b/codalab/worker/main.py index c2ea1790a..e679601bc 100644 --- a/codalab/worker/main.py +++ b/codalab/worker/main.py @@ -326,7 +326,7 @@ def main(): # Create temp file to store kubernetes cert, as we need to pass in a file path. # TODO: Delete the file afterwards (upon CodaLab service stop?) with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: - f.write(args.cert.replace(r'\n', '\n')) + f.write(args.kubernetes_cert.replace(r'\n', '\n')) kubernetes_cert_path = f.name logger.info('Temporarily writing kubernetes cert to: %s', kubernetes_cert_path) else: From bf9f7bd88798a5e94bb28334066897855e2d8432 Mon Sep 17 00:00:00 2001 From: Ashwin Ramaswami Date: Wed, 27 Sep 2023 22:22:08 +0000 Subject: [PATCH 5/7] Use env variables instead --- codalab/worker/main.py | 2 +- .../worker_manager/kubernetes_worker_manager.py | 6 ++++-- docker_config/compose_files/docker-compose.yml | 14 ++++++++++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/codalab/worker/main.py b/codalab/worker/main.py index e679601bc..ba59955cd 100644 --- a/codalab/worker/main.py +++ b/codalab/worker/main.py @@ -326,7 +326,7 @@ def main(): # Create temp file to store kubernetes cert, as we need to pass in a file path. # TODO: Delete the file afterwards (upon CodaLab service stop?) with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: - f.write(args.kubernetes_cert.replace(r'\n', '\n')) + f.write(args.kubernetes_cert) kubernetes_cert_path = f.name logger.info('Temporarily writing kubernetes cert to: %s', kubernetes_cert_path) else: diff --git a/codalab/worker_manager/kubernetes_worker_manager.py b/codalab/worker_manager/kubernetes_worker_manager.py index dc96bc1b4..25182d03c 100644 --- a/codalab/worker_manager/kubernetes_worker_manager.py +++ b/codalab/worker_manager/kubernetes_worker_manager.py @@ -47,12 +47,14 @@ def add_arguments_to_subparser(subparser: ArgumentParser) -> None: type=str, help='Path to the SSL cert for the Kubernetes cluster', required=True, + default=os.getenv('CODALAB_CERT_PATH', '/dev/null'), ) subparser.add_argument( '--cert', type=str, help='Contents of the SSL cert for the Kubernetes cluster', - required=True, + required=False, + default=os.getenv('CODALAB_CERT', '/dev/null'), ) subparser.add_argument( '--nfs-volume-name', type=str, help='Name of the persistent volume for the NFS server.', @@ -94,7 +96,7 @@ def __init__(self, args): # Create temp file to store kubernetes cert, as we need to pass in a file path. # TODO: Delete the file afterwards (upon CodaLab service stop?) with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: - f.write(args.cert.replace(r'\n', '\n')) + f.write(args.cert) cert_path = f.name logger.info('Temporarily writing kubernetes cert to: %s', cert_path) else: diff --git a/docker_config/compose_files/docker-compose.yml b/docker_config/compose_files/docker-compose.yml index 5f266fb2f..3686881a9 100644 --- a/docker_config/compose_files/docker-compose.yml +++ b/docker_config/compose_files/docker-compose.yml @@ -302,12 +302,15 @@ services: --bundle-runtime ${CODALAB_WORKER_MANAGER_CPU_BUNDLE_RUNTIME} --cluster-host ${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CLUSTER_HOST} --auth-token ${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_AUTH_TOKEN} - --cert-path ${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CERT_PATH} - --cert "${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CERT}" --cpus ${CODALAB_WORKER_MANAGER_CPU_DEFAULT_CPUS} --memory-mb ${CODALAB_WORKER_MANAGER_CPU_DEFAULT_MEMORY_MB} <<: *codalab-base <<: *codalab-server + environment: + - CODALAB_USERNAME=${CODALAB_USERNAME} + - CODALAB_PASSWORD=${CODALAB_PASSWORD} + - CODALAB_CERT_PATH=${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT_PATH} + - CODALAB_CERT=${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT} volumes: - "${CODALAB_HOME}:${CODALAB_HOME}" - ${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CERT_PATH}:${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CERT_PATH}:ro @@ -336,13 +339,16 @@ services: --bundle-runtime ${CODALAB_WORKER_MANAGER_GPU_BUNDLE_RUNTIME} --cluster-host ${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CLUSTER_HOST} --auth-token ${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_AUTH_TOKEN} - --cert-path ${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT_PATH} - --cert "${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT}" --cpus ${CODALAB_WORKER_MANAGER_GPU_DEFAULT_CPUS} --gpus ${CODALAB_WORKER_MANAGER_DEFAULT_GPUS} --memory-mb ${CODALAB_WORKER_MANAGER_GPU_DEFAULT_MEMORY_MB} <<: *codalab-base <<: *codalab-server + environment: + - CODALAB_USERNAME=${CODALAB_USERNAME} + - CODALAB_PASSWORD=${CODALAB_PASSWORD} + - CODALAB_CERT_PATH=${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT_PATH} + - CODALAB_CERT=${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT} volumes: - "${CODALAB_HOME}:${CODALAB_HOME}" - ${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT_PATH}:${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT_PATH}:ro From 932f2ecfc4632aeda21455f0736aaf16d31f8ce4 Mon Sep 17 00:00:00 2001 From: Ashwin Ramaswami Date: Wed, 4 Oct 2023 22:10:05 +0000 Subject: [PATCH 6/7] Updates --- codalab/worker_manager/kubernetes_worker_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codalab/worker_manager/kubernetes_worker_manager.py b/codalab/worker_manager/kubernetes_worker_manager.py index 25182d03c..e13923b3a 100644 --- a/codalab/worker_manager/kubernetes_worker_manager.py +++ b/codalab/worker_manager/kubernetes_worker_manager.py @@ -46,7 +46,7 @@ def add_arguments_to_subparser(subparser: ArgumentParser) -> None: '--cert-path', type=str, help='Path to the SSL cert for the Kubernetes cluster', - required=True, + required=False, default=os.getenv('CODALAB_CERT_PATH', '/dev/null'), ) subparser.add_argument( From 984757c82bac49729e5df48b9590df015c2bb34b Mon Sep 17 00:00:00 2001 From: Ashwin Ramaswami Date: Wed, 11 Oct 2023 20:11:49 -0400 Subject: [PATCH 7/7] Update docker-compose.yml --- docker_config/compose_files/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker_config/compose_files/docker-compose.yml b/docker_config/compose_files/docker-compose.yml index 3686881a9..5ee8e88d2 100644 --- a/docker_config/compose_files/docker-compose.yml +++ b/docker_config/compose_files/docker-compose.yml @@ -309,8 +309,8 @@ services: environment: - CODALAB_USERNAME=${CODALAB_USERNAME} - CODALAB_PASSWORD=${CODALAB_PASSWORD} - - CODALAB_CERT_PATH=${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT_PATH} - - CODALAB_CERT=${CODALAB_WORKER_MANAGER_GPU_KUBERNETES_CERT} + - CODALAB_CERT_PATH=${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CERT_PATH} + - CODALAB_CERT=${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CERT} volumes: - "${CODALAB_HOME}:${CODALAB_HOME}" - ${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CERT_PATH}:${CODALAB_WORKER_MANAGER_CPU_KUBERNETES_CERT_PATH}:ro