[grid] Add support Node spawn in K8s cluster Signed-off-by: Viet Nguyen Duc <nguyenducviet4496@gmail.com>
diff --git a/MODULE.bazel b/MODULE.bazel index a0702cf..0b8e59e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -193,6 +193,8 @@ "com.uber.nullaway:nullaway:0.12.13", "dev.failsafe:failsafe:3.3.2", "io.grpc:grpc-context:1.77.0", + "io.kubernetes:client-java:25.0.0", + "io.kubernetes:client-java-api:25.0.0", "io.lettuce:lettuce-core:7.1.0.RELEASE", "io.netty:netty-buffer", "io.netty:netty-codec-http", @@ -246,6 +248,7 @@ "org.hamcrest:hamcrest-all", # Replaced by hamcrest 2 "org.hamcrest:hamcrest-core", "io.netty:netty-all", # Depend on the actual things you need + "javax.annotation:javax.annotation-api", # Conflicts with java.annotation module ], fail_if_repin_required = True, fail_on_missing_checksum = True,
diff --git a/java/maven_install.json b/java/maven_install.json index 0e92b68..db20b2a 100644 --- a/java/maven_install.json +++ b/java/maven_install.json
@@ -1,7 +1,7 @@ { "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL", - "__INPUT_ARTIFACTS_HASH": -2085469941, - "__RESOLVED_ARTIFACTS_HASH": 448870062, + "__INPUT_ARTIFACTS_HASH": 1227809314, + "__RESOLVED_ARTIFACTS_HASH": -474621071, "artifacts": { "com.beust:jcommander": { "shasums": { @@ -205,6 +205,41 @@ }, "version": "5.0.0" }, + "com.squareup.okhttp3:logging-interceptor": { + "shasums": { + "jar": "276ac5f68ea4a07aac7fa7effde756336dc994b76ce5ca8a428598d6c94017bb", + "sources": "3b0ceacc3f71b927cdebe0377114d7aa7ae9cd158f8dcd9ccec29868904690bc" + }, + "version": "5.1.0" + }, + "com.squareup.okhttp3:okhttp": { + "shasums": { + "jar": "e720a383fdc3eb1df4aac77a085e61b7730837364151de867093fdbcafcb44aa", + "sources": "16b1025e2389e538d4e51afeeb422d956dcf54385e40cf029584ce77b87e0f8e" + }, + "version": "5.1.0" + }, + "com.squareup.okhttp3:okhttp-jvm": { + "shasums": { + "jar": "a6aac7f15d3c2c3cbd2af7ecf56f97407164a604b2db879529950e31cea699b2", + "sources": "875918275c8e948892d6aa667e36fbfe2c211fe2bf245fb6702b34363df69129" + }, + "version": "5.1.0" + }, + "com.squareup.okio:okio": { + "shasums": { + "jar": "0ed4125e048c86af7ece446d654f2ba8bcc5e74d0ae1f4de3c5dc0ad9a975e2d", + "sources": "b317ddb4253e2d00a57c0dcd76058d6baeca376724430eacf9f397d90c2dd0df" + }, + "version": "3.15.0" + }, + "com.squareup.okio:okio-jvm": { + "shasums": { + "jar": "483dc383b1049d220892322a0d6a4308425f09bb05b48c43d4eaa9ff82b1cd16", + "sources": "eca487c49a4905a89556ce413ea198f40f20bb2dc6249d916b3917913bd6dbab" + }, + "version": "3.15.0" + }, "com.uber.nullaway:nullaway": { "shasums": { "jar": "50c0367cba92d910e826f0011565bd8963b3111ac1bd6c72b04adb3dcc698a75", @@ -212,12 +247,19 @@ }, "version": "0.12.13" }, + "commons-codec:commons-codec": { + "shasums": { + "jar": "5c3881e4f556855e9c532927ee0c9dfde94cc66760d5805c031a59887070af5f", + "sources": "b0462142585d45fc15bc8091b7b02f1e3a85c83595068659548c82cac9cdc7a2" + }, + "version": "1.19.0" + }, "commons-io:commons-io": { "shasums": { - "jar": "a10418348d234968600ccb1d988efcbbd08716e1d96936ccc1880e7d22513474", - "sources": "3b69b518d9a844732e35509b79e499fca63a960ee4301b1c96dc32e87f3f60a1" + "jar": "df90bba0fe3cb586b7f164e78fe8f8f4da3f2dd5c27fa645f888100ccc25dd72", + "sources": "7a87277538cce40da6389a7163a4d9458bc7a9c39937a329881b91d144be8e0d" }, - "version": "2.5" + "version": "2.20.0" }, "commons-logging:commons-logging": { "shasums": { @@ -254,6 +296,34 @@ }, "version": "1.77.0" }, + "io.gsonfire:gson-fire": { + "shasums": { + "jar": "73f56642ef43381efda085befb1e6cb51ce906af426d8db19632d396a5bb7a91", + "sources": "40d8500f33c7309515782d5381593a9d6c42699fc3fe38df0055ec4e08055f59" + }, + "version": "1.9.0" + }, + "io.kubernetes:client-java": { + "shasums": { + "jar": "8fe03b28239bc59959c828a0a7c38a2677c2b07b9f22eefeeb44c198dd6f70c5", + "sources": "5fb42785723f689d95e308c44f31a29ed01cf140bc29385fc20ae631f433899c" + }, + "version": "25.0.0" + }, + "io.kubernetes:client-java-api": { + "shasums": { + "jar": "d5bde184fbd0d820ee5207065004c4c143a2beaab16f2f5137ce80cf5880ab40", + "sources": "c65ef7229851d25f035d3d9e3fe52ad61b37e8e31f2dcfbb7f458763b9e17b32" + }, + "version": "25.0.0" + }, + "io.kubernetes:client-java-proto": { + "shasums": { + "jar": "1efd70b8e086428307e81d371021daf2a8b1835605f9cb2546d467b107c65d6e", + "sources": "9e33d4d795cc035460b76090613bf67c1b3ef4d10c6af9bf393d1c1d730c4398" + }, + "version": "25.0.0" + }, "io.lettuce:lettuce-core": { "shasums": { "jar": "d974ffdb0452e675985ff89b3c33f43ec9bc881f1cb1f7d140cfb303f8943ae6", @@ -478,6 +548,13 @@ }, "version": "3.1.8" }, + "io.swagger:swagger-annotations": { + "shasums": { + "jar": "c832295d639aa54139404b7406fb2f8fbf1da8c57219df3395de475503464297", + "sources": "7b2de9dc92520bd12e30987ee491191131e719d30e3bbe8a536f18e80ca14df3" + }, + "version": "1.6.16" + }, "it.ozimov:embedded-redis": { "shasums": { "jar": "f655f2ece0bb01b4d28b937877d4287d12bd7199e7cee83b00b1653dba00c1df", @@ -485,6 +562,13 @@ }, "version": "0.7.3" }, + "jakarta.annotation:jakarta.annotation-api": { + "shasums": { + "jar": "b01f55552284cfb149411e64eabca75e942d26d2e1786b32914250e4330afaa2", + "sources": "142dfd2343429df2aac3e1cfacfacc21c8393112c0e0280d3628648d9b612470" + }, + "version": "3.0.0" + }, "javax.cache:cache-api": { "shasums": { "jar": "9f34e007edfa82a7b2a2e1b969477dcf5099ce7f4f926fb54ce7e27c4a0cd54b", @@ -534,6 +618,20 @@ }, "version": "6.11.0" }, + "org.apache.commons:commons-collections4": { + "shasums": { + "jar": "00f93263c267be201b8ae521b44a7137271b16688435340bf629db1bac0a5845", + "sources": "75f1bef9447cce189743f7d52f63a669bd796ae19ca863e1f22db1d5b6b504a6" + }, + "version": "4.5.0" + }, + "org.apache.commons:commons-compress": { + "shasums": { + "jar": "e1522945218456f3649a39bc4afd70ce4bd466221519dba7d378f2141a4642ca", + "sources": "6de9de4559f12bba6d41789c72f6a2a424514f2d2a3f7f49e2a3c52414db9632" + }, + "version": "1.28.0" + }, "org.apache.commons:commons-exec": { "shasums": { "jar": "13dcf3850478ef8de5d24d298a60eed5e8305eb20538fe632c82ea1dff6b5ea0", @@ -583,6 +681,13 @@ }, "version": "3.27.6" }, + "org.bitbucket.b_c:jose4j": { + "shasums": { + "jar": "7314af50cde9c99e8eaf43eee617a23edcc6bb43036221064355094999d837ef", + "sources": "958be1837b507d3a1f1187257072b4c1e1d031c3a0c610d3c02ac69aabfac6a5" + }, + "version": "0.9.6" + }, "org.bouncycastle:bcpkix-jdk18on": { "shasums": { "jar": "d3c4c6b700c74ef8164bb15e549d939721b8f14fc0ff89fe19b220243bcfcbd8", @@ -653,6 +758,20 @@ }, "version": "2.2.3.Final" }, + "org.jetbrains.kotlin:kotlin-stdlib": { + "shasums": { + "jar": "65d12d85a3b865c160db9147851712a64b10dadd68b22eea22a95bf8a8670dca", + "sources": "967ad9599254e3a60d96d6c789547cc35c22d770d9c8fb1e3f15fac3b4c3b65d" + }, + "version": "2.2.0" + }, + "org.jetbrains:annotations": { + "shasums": { + "jar": "ace2a10dc8e2d5fd34925ecac03e4988b2c0f851650c94b8cef49ba1bd111478", + "sources": "42a5e144b8e81d50d6913d1007b695e62e614705268d8cf9f13dbdc478c2c68e" + }, + "version": "13.0" + }, "org.jodd:jodd-util": { "shasums": { "jar": "e5c676715897124101f74900dd1f98ebbcad9da3ab2dffbac6e7712612094427", @@ -831,10 +950,10 @@ }, "org.yaml:snakeyaml": { "shasums": { - "jar": "ef779af5d29a9dde8cc70ce0341f5c6f7735e23edff9685ceaa9d35359b7bb7f", - "sources": "0a3c537e86cbb13a9a31ab138c13cee0b2c2cedc17dba38e0dc1f24a2053d8af" + "jar": "e6682acf1ace77508ef13649cbf4f8d09d2cf5457bdb61d25ffb6ac0233d78dd", + "sources": "7a7d307ca9fe1663219a60045a8e5a113a69331566f9ebbe1d3b12c6781909ac" }, - "version": "2.4" + "version": "2.5" }, "org.zeromq:jeromq": { "shasums": { @@ -867,12 +986,14 @@ }, "conflict_resolution": { "com.google.errorprone:error_prone_annotations:2.43.0": "com.google.errorprone:error_prone_annotations:2.41.0", - "commons-io:commons-io:2.20.0": "commons-io:commons-io:2.5", + "commons-io:commons-io:2.5": "commons-io:commons-io:2.20.0", "io.projectreactor:reactor-core:3.6.2": "io.projectreactor:reactor-core:3.6.6", "net.bytebuddy:byte-buddy-agent:1.17.4": "net.bytebuddy:byte-buddy-agent:1.17.7", "org.apache.commons:commons-lang3:3.18.0": "org.apache.commons:commons-lang3:3.19.0", + "org.jetbrains.kotlin:kotlin-stdlib:2.1.21": "org.jetbrains.kotlin:kotlin-stdlib:2.2.0", "org.objenesis:objenesis:3.4": "org.objenesis:objenesis:3.3", - "org.reactivestreams:reactive-streams:1.0.4": "org.reactivestreams:reactive-streams:1.0.3" + "org.reactivestreams:reactive-streams:1.0.4": "org.reactivestreams:reactive-streams:1.0.3", + "org.yaml:snakeyaml:2.4": "org.yaml:snakeyaml:2.5" }, "dependencies": { "com.esotericsoftware:kryo": [ @@ -949,6 +1070,24 @@ "org.jspecify:jspecify", "org.reactivestreams:reactive-streams" ], + "com.squareup.okhttp3:logging-interceptor": [ + "com.squareup.okhttp3:okhttp-jvm", + "org.jetbrains.kotlin:kotlin-stdlib" + ], + "com.squareup.okhttp3:okhttp": [ + "com.squareup.okio:okio", + "org.jetbrains.kotlin:kotlin-stdlib" + ], + "com.squareup.okhttp3:okhttp-jvm": [ + "com.squareup.okio:okio-jvm", + "org.jetbrains.kotlin:kotlin-stdlib" + ], + "com.squareup.okio:okio": [ + "com.squareup.okio:okio-jvm" + ], + "com.squareup.okio:okio-jvm": [ + "org.jetbrains.kotlin:kotlin-stdlib" + ], "com.uber.nullaway:nullaway": [ "com.google.guava:guava", "org.checkerframework:dataflow-nullaway", @@ -957,6 +1096,37 @@ "io.grpc:grpc-context": [ "io.grpc:grpc-api" ], + "io.gsonfire:gson-fire": [ + "com.google.code.gson:gson" + ], + "io.kubernetes:client-java": [ + "com.google.protobuf:protobuf-java", + "commons-codec:commons-codec", + "commons-io:commons-io", + "io.kubernetes:client-java-api", + "io.kubernetes:client-java-proto", + "org.apache.commons:commons-collections4", + "org.apache.commons:commons-compress", + "org.apache.commons:commons-lang3", + "org.bitbucket.b_c:jose4j", + "org.bouncycastle:bcpkix-jdk18on", + "org.slf4j:slf4j-api", + "org.yaml:snakeyaml" + ], + "io.kubernetes:client-java-api": [ + "com.fasterxml.jackson.core:jackson-databind", + "com.google.code.findbugs:jsr305", + "com.google.code.gson:gson", + "com.squareup.okhttp3:logging-interceptor", + "com.squareup.okhttp3:okhttp", + "io.gsonfire:gson-fire", + "io.swagger:swagger-annotations", + "jakarta.annotation:jakarta.annotation-api", + "org.apache.commons:commons-lang3" + ], + "io.kubernetes:client-java-proto": [ + "com.google.protobuf:protobuf-java" + ], "io.lettuce:lettuce-core": [ "io.netty:netty-common", "io.netty:netty-handler", @@ -1130,6 +1300,11 @@ "commons-io:commons-io", "org.apache.commons:commons-lang3" ], + "org.apache.commons:commons-compress": [ + "commons-codec:commons-codec", + "commons-io:commons-io", + "org.apache.commons:commons-lang3" + ], "org.apache.commons:commons-text": [ "org.apache.commons:commons-lang3" ], @@ -1139,6 +1314,9 @@ "org.assertj:assertj-core": [ "net.bytebuddy:byte-buddy" ], + "org.bitbucket.b_c:jose4j": [ + "org.slf4j:slf4j-api" + ], "org.bouncycastle:bcpkix-jdk18on": [ "org.bouncycastle:bcutil-jdk18on" ], @@ -1151,6 +1329,9 @@ "org.eclipse.mylyn.github:org.eclipse.egit.github.core": [ "com.google.code.gson:gson" ], + "org.jetbrains.kotlin:kotlin-stdlib": [ + "org.jetbrains:annotations" + ], "org.junit.jupiter:junit-jupiter-api": [ "org.apiguardian:apiguardian-api", "org.jspecify:jspecify", @@ -1581,6 +1762,35 @@ "org.dataloader.stats", "org.dataloader.stats.context" ], + "com.squareup.okhttp3:logging-interceptor": [ + "okhttp3.logging", + "okhttp3.logging.internal" + ], + "com.squareup.okhttp3:okhttp-jvm": [ + "okhttp3", + "okhttp3.internal", + "okhttp3.internal.authenticator", + "okhttp3.internal.cache", + "okhttp3.internal.cache2", + "okhttp3.internal.concurrent", + "okhttp3.internal.connection", + "okhttp3.internal.graal", + "okhttp3.internal.http", + "okhttp3.internal.http1", + "okhttp3.internal.http2", + "okhttp3.internal.http2.flowcontrol", + "okhttp3.internal.idn", + "okhttp3.internal.platform", + "okhttp3.internal.proxy", + "okhttp3.internal.publicsuffix", + "okhttp3.internal.tls", + "okhttp3.internal.url", + "okhttp3.internal.ws" + ], + "com.squareup.okio:okio-jvm": [ + "okio", + "okio.internal" + ], "com.uber.nullaway:nullaway": [ "com.uber.nullaway", "com.uber.nullaway.annotations", @@ -1598,11 +1808,28 @@ "com.uber.nullaway.handlers.temporary", "com.uber.nullaway.jarinfer" ], + "commons-codec:commons-codec": [ + "org.apache.commons.codec", + "org.apache.commons.codec.binary", + "org.apache.commons.codec.cli", + "org.apache.commons.codec.digest", + "org.apache.commons.codec.language", + "org.apache.commons.codec.language.bm", + "org.apache.commons.codec.net" + ], "commons-io:commons-io": [ "org.apache.commons.io", + "org.apache.commons.io.build", + "org.apache.commons.io.channels", + "org.apache.commons.io.charset", "org.apache.commons.io.comparator", + "org.apache.commons.io.file", + "org.apache.commons.io.file.attribute", + "org.apache.commons.io.file.spi", "org.apache.commons.io.filefilter", + "org.apache.commons.io.function", "org.apache.commons.io.input", + "org.apache.commons.io.input.buffer", "org.apache.commons.io.monitor", "org.apache.commons.io.output", "org.apache.commons.io.serialization" @@ -1626,6 +1853,53 @@ "io.grpc:grpc-api": [ "io.grpc" ], + "io.gsonfire:gson-fire": [ + "io.gsonfire", + "io.gsonfire.annotations", + "io.gsonfire.builders", + "io.gsonfire.gson", + "io.gsonfire.postprocessors", + "io.gsonfire.postprocessors.methodinvoker", + "io.gsonfire.util", + "io.gsonfire.util.reflection" + ], + "io.kubernetes:client-java": [ + "io.kubernetes.client", + "io.kubernetes.client.apimachinery", + "io.kubernetes.client.informer", + "io.kubernetes.client.informer.cache", + "io.kubernetes.client.informer.exception", + "io.kubernetes.client.informer.impl", + "io.kubernetes.client.monitoring", + "io.kubernetes.client.persister", + "io.kubernetes.client.simplified", + "io.kubernetes.client.util", + "io.kubernetes.client.util.annotations", + "io.kubernetes.client.util.authenticators", + "io.kubernetes.client.util.conversion", + "io.kubernetes.client.util.credentials", + "io.kubernetes.client.util.exception", + "io.kubernetes.client.util.generic", + "io.kubernetes.client.util.generic.dynamic", + "io.kubernetes.client.util.generic.options", + "io.kubernetes.client.util.labels", + "io.kubernetes.client.util.okhttp", + "io.kubernetes.client.util.taints", + "io.kubernetes.client.util.version", + "io.kubernetes.client.util.wait" + ], + "io.kubernetes:client-java-api": [ + "io.kubernetes.client.common", + "io.kubernetes.client.custom", + "io.kubernetes.client.gson", + "io.kubernetes.client.openapi", + "io.kubernetes.client.openapi.apis", + "io.kubernetes.client.openapi.auth", + "io.kubernetes.client.openapi.models" + ], + "io.kubernetes:client-java-proto": [ + "io.kubernetes.client.proto" + ], "io.lettuce:lettuce-core": [ "io.lettuce.authx", "io.lettuce.core", @@ -1938,12 +2212,20 @@ "io.reactivex.rxjava3.subjects", "io.reactivex.rxjava3.subscribers" ], + "io.swagger:swagger-annotations": [ + "io.swagger.annotations" + ], "it.ozimov:embedded-redis": [ "redis.embedded", "redis.embedded.exceptions", "redis.embedded.ports", "redis.embedded.util" ], + "jakarta.annotation:jakarta.annotation-api": [ + "jakarta.annotation", + "jakarta.annotation.security", + "jakarta.annotation.sql" + ], "javax.cache:cache-api": [ "javax.cache", "javax.cache.annotation", @@ -2099,6 +2381,66 @@ "org.apache.bcel.verifier.statics", "org.apache.bcel.verifier.structurals" ], + "org.apache.commons:commons-collections4": [ + "org.apache.commons.collections4", + "org.apache.commons.collections4.bag", + "org.apache.commons.collections4.bidimap", + "org.apache.commons.collections4.bloomfilter", + "org.apache.commons.collections4.collection", + "org.apache.commons.collections4.comparators", + "org.apache.commons.collections4.functors", + "org.apache.commons.collections4.iterators", + "org.apache.commons.collections4.keyvalue", + "org.apache.commons.collections4.list", + "org.apache.commons.collections4.map", + "org.apache.commons.collections4.multimap", + "org.apache.commons.collections4.multiset", + "org.apache.commons.collections4.properties", + "org.apache.commons.collections4.queue", + "org.apache.commons.collections4.sequence", + "org.apache.commons.collections4.set", + "org.apache.commons.collections4.splitmap", + "org.apache.commons.collections4.trie", + "org.apache.commons.collections4.trie.analyzer" + ], + "org.apache.commons:commons-compress": [ + "org.apache.commons.compress", + "org.apache.commons.compress.archivers", + "org.apache.commons.compress.archivers.ar", + "org.apache.commons.compress.archivers.arj", + "org.apache.commons.compress.archivers.cpio", + "org.apache.commons.compress.archivers.dump", + "org.apache.commons.compress.archivers.examples", + "org.apache.commons.compress.archivers.jar", + "org.apache.commons.compress.archivers.sevenz", + "org.apache.commons.compress.archivers.tar", + "org.apache.commons.compress.archivers.zip", + "org.apache.commons.compress.changes", + "org.apache.commons.compress.compressors", + "org.apache.commons.compress.compressors.brotli", + "org.apache.commons.compress.compressors.bzip2", + "org.apache.commons.compress.compressors.deflate", + "org.apache.commons.compress.compressors.deflate64", + "org.apache.commons.compress.compressors.gzip", + "org.apache.commons.compress.compressors.lz4", + "org.apache.commons.compress.compressors.lz77support", + "org.apache.commons.compress.compressors.lzma", + "org.apache.commons.compress.compressors.lzw", + "org.apache.commons.compress.compressors.pack200", + "org.apache.commons.compress.compressors.snappy", + "org.apache.commons.compress.compressors.xz", + "org.apache.commons.compress.compressors.z", + "org.apache.commons.compress.compressors.zstandard", + "org.apache.commons.compress.harmony", + "org.apache.commons.compress.harmony.archive.internal.nls", + "org.apache.commons.compress.harmony.pack200", + "org.apache.commons.compress.harmony.unpack200", + "org.apache.commons.compress.harmony.unpack200.bytecode", + "org.apache.commons.compress.harmony.unpack200.bytecode.forms", + "org.apache.commons.compress.java.util.jar", + "org.apache.commons.compress.parallel", + "org.apache.commons.compress.utils" + ], "org.apache.commons:commons-exec": [ "org.apache.commons.exec", "org.apache.commons.exec.environment", @@ -2241,6 +2583,28 @@ "org.assertj.core.util.introspection", "org.assertj.core.util.xml" ], + "org.bitbucket.b_c:jose4j": [ + "org.jose4j.base64url", + "org.jose4j.base64url.internal.apache.commons.codec.binary", + "org.jose4j.http", + "org.jose4j.jca", + "org.jose4j.json", + "org.jose4j.json.internal.json_simple", + "org.jose4j.json.internal.json_simple.parser", + "org.jose4j.jwa", + "org.jose4j.jwe", + "org.jose4j.jwe.kdf", + "org.jose4j.jwk", + "org.jose4j.jws", + "org.jose4j.jwt", + "org.jose4j.jwt.consumer", + "org.jose4j.jwx", + "org.jose4j.keys", + "org.jose4j.keys.resolvers", + "org.jose4j.lang", + "org.jose4j.mac", + "org.jose4j.zip" + ], "org.bouncycastle:bcpkix-jdk18on": [ "org.bouncycastle.cert", "org.bouncycastle.cert.bc", @@ -2687,6 +3051,58 @@ "org.jboss.marshalling.reflect", "org.jboss.marshalling.util" ], + "org.jetbrains.kotlin:kotlin-stdlib": [ + "kotlin", + "kotlin.annotation", + "kotlin.collections", + "kotlin.collections.builders", + "kotlin.collections.jdk8", + "kotlin.collections.unsigned", + "kotlin.comparisons", + "kotlin.concurrent", + "kotlin.concurrent.atomics", + "kotlin.concurrent.internal", + "kotlin.contracts", + "kotlin.coroutines", + "kotlin.coroutines.cancellation", + "kotlin.coroutines.intrinsics", + "kotlin.coroutines.jvm.internal", + "kotlin.enums", + "kotlin.experimental", + "kotlin.internal", + "kotlin.internal.jdk7", + "kotlin.internal.jdk8", + "kotlin.io", + "kotlin.io.encoding", + "kotlin.io.path", + "kotlin.jdk7", + "kotlin.js", + "kotlin.jvm", + "kotlin.jvm.functions", + "kotlin.jvm.internal", + "kotlin.jvm.internal.markers", + "kotlin.jvm.internal.unsafe", + "kotlin.jvm.jdk8", + "kotlin.jvm.optionals", + "kotlin.math", + "kotlin.properties", + "kotlin.random", + "kotlin.random.jdk8", + "kotlin.ranges", + "kotlin.reflect", + "kotlin.sequences", + "kotlin.streams.jdk8", + "kotlin.system", + "kotlin.text", + "kotlin.text.jdk8", + "kotlin.time", + "kotlin.time.jdk8", + "kotlin.uuid" + ], + "org.jetbrains:annotations": [ + "org.intellij.lang.annotations", + "org.jetbrains.annotations" + ], "org.jodd:jodd-util": [ "jodd", "jodd.bean", @@ -3113,8 +3529,20 @@ "com.graphql-java:graphql-java:jar:sources", "com.graphql-java:java-dataloader", "com.graphql-java:java-dataloader:jar:sources", + "com.squareup.okhttp3:logging-interceptor", + "com.squareup.okhttp3:logging-interceptor:jar:sources", + "com.squareup.okhttp3:okhttp", + "com.squareup.okhttp3:okhttp-jvm", + "com.squareup.okhttp3:okhttp-jvm:jar:sources", + "com.squareup.okhttp3:okhttp:jar:sources", + "com.squareup.okio:okio", + "com.squareup.okio:okio-jvm", + "com.squareup.okio:okio-jvm:jar:sources", + "com.squareup.okio:okio:jar:sources", "com.uber.nullaway:nullaway", "com.uber.nullaway:nullaway:jar:sources", + "commons-codec:commons-codec", + "commons-codec:commons-codec:jar:sources", "commons-io:commons-io", "commons-io:commons-io:jar:sources", "commons-logging:commons-logging", @@ -3127,6 +3555,14 @@ "io.grpc:grpc-api:jar:sources", "io.grpc:grpc-context", "io.grpc:grpc-context:jar:sources", + "io.gsonfire:gson-fire", + "io.gsonfire:gson-fire:jar:sources", + "io.kubernetes:client-java", + "io.kubernetes:client-java-api", + "io.kubernetes:client-java-api:jar:sources", + "io.kubernetes:client-java-proto", + "io.kubernetes:client-java-proto:jar:sources", + "io.kubernetes:client-java:jar:sources", "io.lettuce:lettuce-core", "io.lettuce:lettuce-core:jar:sources", "io.netty:netty-buffer", @@ -3191,8 +3627,12 @@ "io.projectreactor:reactor-core:jar:sources", "io.reactivex.rxjava3:rxjava", "io.reactivex.rxjava3:rxjava:jar:sources", + "io.swagger:swagger-annotations", + "io.swagger:swagger-annotations:jar:sources", "it.ozimov:embedded-redis", "it.ozimov:embedded-redis:jar:sources", + "jakarta.annotation:jakarta.annotation-api", + "jakarta.annotation:jakarta.annotation-api:jar:sources", "javax.cache:cache-api", "javax.cache:cache-api:jar:sources", "jaxen:jaxen", @@ -3207,6 +3647,10 @@ "org.antlr:antlr4-runtime:jar:sources", "org.apache.bcel:bcel", "org.apache.bcel:bcel:jar:sources", + "org.apache.commons:commons-collections4", + "org.apache.commons:commons-collections4:jar:sources", + "org.apache.commons:commons-compress", + "org.apache.commons:commons-compress:jar:sources", "org.apache.commons:commons-exec", "org.apache.commons:commons-exec:jar:sources", "org.apache.commons:commons-lang3", @@ -3221,6 +3665,8 @@ "org.apiguardian:apiguardian-api:jar:sources", "org.assertj:assertj-core", "org.assertj:assertj-core:jar:sources", + "org.bitbucket.b_c:jose4j", + "org.bitbucket.b_c:jose4j:jar:sources", "org.bouncycastle:bcpkix-jdk18on", "org.bouncycastle:bcpkix-jdk18on:jar:sources", "org.bouncycastle:bcprov-jdk18on", @@ -3241,6 +3687,10 @@ "org.htmlunit:htmlunit-core-js:jar:sources", "org.jboss.marshalling:jboss-marshalling", "org.jboss.marshalling:jboss-marshalling:jar:sources", + "org.jetbrains.kotlin:kotlin-stdlib", + "org.jetbrains.kotlin:kotlin-stdlib:jar:sources", + "org.jetbrains:annotations", + "org.jetbrains:annotations:jar:sources", "org.jodd:jodd-util", "org.jodd:jodd-util:jar:sources", "org.jspecify:jspecify",
diff --git a/java/src/org/openqa/selenium/grid/node/k8s/BUILD.bazel b/java/src/org/openqa/selenium/grid/node/k8s/BUILD.bazel index 68b5f9c..627a6b5 100644 --- a/java/src/org/openqa/selenium/grid/node/k8s/BUILD.bazel +++ b/java/src/org/openqa/selenium/grid/node/k8s/BUILD.bazel
@@ -6,13 +6,28 @@ srcs = glob(["*.java"]), visibility = [ "//deploys/docker:__pkg__", + "//java/src/org/openqa/selenium/grid/commands:__pkg__", + "//java/src/org/openqa/selenium/grid/node/httpd:__pkg__", + "//java/src/org/openqa/selenium/grid/node/local:__pkg__", + "//java/test/org/openqa/selenium/grid/node/k8s:__pkg__", ], deps = [ "//java:auto-service", - "//java/src/org/openqa/selenium/grid", + "//java/src/org/openqa/selenium/events", + "//java/src/org/openqa/selenium/grid/config", + "//java/src/org/openqa/selenium/grid/data", + "//java/src/org/openqa/selenium/grid/jmx", + "//java/src/org/openqa/selenium/grid/log", + "//java/src/org/openqa/selenium/grid/node", + "//java/src/org/openqa/selenium/grid/node/config", + "//java/src/org/openqa/selenium/grid/security", + "//java/src/org/openqa/selenium/grid/server", "//java/src/org/openqa/selenium/json", "//java/src/org/openqa/selenium/remote", + "//java/src/org/openqa/selenium/support", + "@maven//:io_kubernetes_client_java_api", artifact("com.beust:jcommander"), artifact("com.google.guava:guava"), + artifact("io.kubernetes:client-java"), ], )
diff --git a/java/src/org/openqa/selenium/grid/node/k8s/KubernetesAssetsPath.java b/java/src/org/openqa/selenium/grid/node/k8s/KubernetesAssetsPath.java new file mode 100644 index 0000000..43cdb49 --- /dev/null +++ b/java/src/org/openqa/selenium/grid/node/k8s/KubernetesAssetsPath.java
@@ -0,0 +1,44 @@ +// Licensed to the Software Freedom Conservancy (SFC) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The SFC licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.openqa.selenium.grid.node.k8s; + +import java.io.File; +import org.openqa.selenium.remote.SessionId; + +public class KubernetesAssetsPath { + + private final String hostPath; + private final String podPath; + + public KubernetesAssetsPath(String hostPath, String podPath) { + this.hostPath = hostPath; + this.podPath = podPath; + } + + public String getHostPath() { + return this.hostPath; + } + + public String getHostPath(SessionId id) { + return this.hostPath + File.separator + id; + } + + public String getPodPath(SessionId id) { + return this.podPath + File.separator + id; + } +}
diff --git a/java/src/org/openqa/selenium/grid/node/k8s/KubernetesFlags.java b/java/src/org/openqa/selenium/grid/node/k8s/KubernetesFlags.java new file mode 100644 index 0000000..c55b151 --- /dev/null +++ b/java/src/org/openqa/selenium/grid/node/k8s/KubernetesFlags.java
@@ -0,0 +1,186 @@ +// Licensed to the Software Freedom Conservancy (SFC) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The SFC licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.openqa.selenium.grid.node.k8s; + +import static org.openqa.selenium.grid.config.StandardGridRoles.NODE_ROLE; + +import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.openqa.selenium.grid.config.ConfigValue; +import org.openqa.selenium.grid.config.HasRoles; +import org.openqa.selenium.grid.config.NonSplittingSplitter; +import org.openqa.selenium.grid.config.Role; + +@SuppressWarnings("FieldMayBeFinal") +@AutoService(HasRoles.class) +public class KubernetesFlags implements HasRoles { + + @Parameter( + names = {"--k8s-kubeconfig"}, + description = "Path to kubeconfig file for Kubernetes cluster access") + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "kubeconfig", + example = "\"/home/user/.kube/config\"") + private String kubeconfig; + + @Parameter( + names = {"--k8s-namespace"}, + description = "Kubernetes namespace for creating browser Jobs/Pods") + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "namespace", + example = "\"" + KubernetesOptions.DEFAULT_NAMESPACE + "\"") + private String namespace; + + @Parameter( + names = {"--k8s-server-start-timeout"}, + description = + "Max time (in seconds) to wait for the browser pod to successfully start up") + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "server-start-timeout", + example = "60") + private Integer serverStartTimeout; + + @Parameter( + names = {"--k8s", "-K"}, + description = + "Kubernetes configs which map image name to stereotype capabilities (example: " + + "-K selenium/standalone-firefox:latest '{\"browserName\": \"firefox\"}')", + arity = 2, + variableArity = true, + splitter = NonSplittingSplitter.class) + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "configs", + example = + "[\"selenium/standalone-firefox:latest\", \"{\\\"browserName\\\": \\\"firefox\\\"}\"]") + private List<String> images2Capabilities; + + @Parameter( + names = {"--k8s-cpu-request"}, + description = "CPU resource request for browser pods (e.g., '500m', '1')") + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "cpu-request", + example = "\"500m\"") + private String cpuRequest; + + @Parameter( + names = {"--k8s-memory-request"}, + description = "Memory resource request for browser pods (e.g., '512Mi', '1Gi')") + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "memory-request", + example = "\"1Gi\"") + private String memoryRequest; + + @Parameter( + names = {"--k8s-cpu-limit"}, + description = "CPU resource limit for browser pods (e.g., '1', '2')") + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "cpu-limit", + example = "\"1\"") + private String cpuLimit; + + @Parameter( + names = {"--k8s-memory-limit"}, + description = "Memory resource limit for browser pods (e.g., '1Gi', '2Gi')") + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "memory-limit", + example = "\"2Gi\"") + private String memoryLimit; + + @Parameter( + names = {"--k8s-labels"}, + description = "Custom labels to apply to browser Jobs/Pods (key-value pairs)", + arity = 2, + variableArity = true, + splitter = NonSplittingSplitter.class) + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "labels", + example = "[\"environment\", \"production\", \"team\", \"qa\"]") + private List<String> labels; + + @Parameter( + names = {"--k8s-annotations"}, + description = "Custom annotations to apply to browser Jobs/Pods (key-value pairs)", + arity = 2, + variableArity = true, + splitter = NonSplittingSplitter.class) + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "annotations", + example = "[\"prometheus.io/scrape\", \"true\"]") + private List<String> annotations; + + @Parameter( + names = {"--k8s-video-image"}, + description = "Docker/Container image to be used when video recording is enabled") + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "video-image", + example = "\"selenium/video:latest\"") + private String videoImage = KubernetesOptions.DEFAULT_VIDEO_IMAGE; + + @Parameter( + names = {"--k8s-assets-path"}, + description = "Absolute path where session assets (logs, videos) will be stored") + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "assets-path", + example = "\"" + KubernetesOptions.DEFAULT_ASSETS_PATH + "\"") + private String assetsPath; + + @Parameter( + names = {"--k8s-image-pull-policy"}, + description = "Image pull policy for browser containers (Always, IfNotPresent, Never)") + @ConfigValue( + section = KubernetesOptions.KUBERNETES_SECTION, + name = "image-pull-policy", + example = "\"IfNotPresent\"") + private String imagePullPolicy; + + // Kept from original OneShotFlags for backward compatibility + @Parameter( + names = {"--driver-name"}, + description = "Name of the browser to use (optional, for OneShotNode)") + @ConfigValue(section = "k8s", name = "driver_name", example = "firefox") + private String driverBinary; + + @Parameter( + names = {"--stereotype"}, + description = "Stringified JSON representing browser stereotype (for OneShotNode)") + @ConfigValue( + section = "k8s", + name = "stereotype", + example = "\"{\\\"browserName\\\": \\\"firefox\\\"}\"") + private String stereotype; + + @Override + public Set<Role> getRoles() { + return Collections.singleton(NODE_ROLE); + } +}
diff --git a/java/src/org/openqa/selenium/grid/node/k8s/KubernetesOptions.java b/java/src/org/openqa/selenium/grid/node/k8s/KubernetesOptions.java new file mode 100644 index 0000000..96ab36a --- /dev/null +++ b/java/src/org/openqa/selenium/grid/node/k8s/KubernetesOptions.java
@@ -0,0 +1,221 @@ +// Licensed to the Software Freedom Conservancy (SFC) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The SFC licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.openqa.selenium.grid.node.k8s; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.Multimap; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.util.Config; +import java.io.IOException; +import java.time.Duration; +import java.util.*; +import java.util.logging.Logger; +import org.openqa.selenium.Capabilities; +import org.openqa.selenium.grid.config.ConfigException; +import org.openqa.selenium.grid.node.SessionFactory; +import org.openqa.selenium.grid.node.config.NodeOptions; +import org.openqa.selenium.internal.Require; +import org.openqa.selenium.json.Json; +import org.openqa.selenium.remote.http.HttpClient; +import org.openqa.selenium.remote.tracing.Tracer; + +public class KubernetesOptions { + + static final String KUBERNETES_SECTION = "k8s"; + static final String DEFAULT_ASSETS_PATH = "/opt/selenium/assets"; + static final String DEFAULT_NAMESPACE = "default"; + static final String DEFAULT_VIDEO_IMAGE = "false"; + static final int DEFAULT_MAX_SESSIONS = Runtime.getRuntime().availableProcessors(); + static final int DEFAULT_SERVER_START_TIMEOUT = 60; + + private static final Logger LOG = Logger.getLogger(KubernetesOptions.class.getName()); + private static final Json JSON = new Json(); + + private final org.openqa.selenium.grid.config.Config config; + + public KubernetesOptions(org.openqa.selenium.grid.config.Config config) { + this.config = Require.nonNull("Config", config); + } + + private ApiClient getKubernetesClient() { + try { + // Try to load from kubeconfig file or in-cluster config + Optional<String> kubeconfigPath = config.get(KUBERNETES_SECTION, "kubeconfig"); + if (kubeconfigPath.isPresent()) { + return Config.fromConfig(kubeconfigPath.get()); + } else { + // Use default kubeconfig or in-cluster configuration + return Config.defaultClient(); + } + } catch (IOException e) { + throw new ConfigException("Unable to create Kubernetes client", e); + } + } + + private String getNamespace() { + return config.get(KUBERNETES_SECTION, "namespace").orElse(DEFAULT_NAMESPACE); + } + + private Duration getServerStartTimeout() { + return Duration.ofSeconds( + config.getInt(KUBERNETES_SECTION, "server-start-timeout").orElse(DEFAULT_SERVER_START_TIMEOUT)); + } + + private boolean isEnabled() { + return config.getAll(KUBERNETES_SECTION, "configs").isPresent(); + } + + public Map<Capabilities, Collection<SessionFactory>> getKubernetesSessionFactories( + Tracer tracer, HttpClient.Factory clientFactory, NodeOptions options) { + + if (!isEnabled()) { + return Collections.emptyMap(); + } + + ApiClient kubernetesClient = getKubernetesClient(); + String namespace = getNamespace(); + + List<String> allConfigs = + config + .getAll(KUBERNETES_SECTION, "configs") + .orElseThrow(() -> new ConfigException("Unable to find kubernetes configs")); + + Multimap<String, Capabilities> kinds = HashMultimap.create(); + int configsCount = allConfigs.size(); + for (int i = 0; i < configsCount; i++) { + String imageName = allConfigs.get(i); + i++; + if (i == configsCount) { + throw new ConfigException("Unable to find JSON config for image: " + imageName); + } + Capabilities stereotype = + options.enhanceStereotype(JSON.toType(allConfigs.get(i), Capabilities.class)); + + kinds.put(imageName, stereotype); + } + + KubernetesAssetsPath assetsPath = getAssetsPath(); + String videoImage = getVideoImage(); + Map<String, String> resourceLimits = getResourceLimits(); + Map<String, String> resourceRequests = getResourceRequests(); + Map<String, String> labels = getLabels(); + Map<String, String> annotations = getAnnotations(); + + int maxSessionCount = + Math.min( + config.getInt("node", "max-sessions").orElse(DEFAULT_MAX_SESSIONS), + DEFAULT_MAX_SESSIONS); + + ImmutableMultimap.Builder<Capabilities, SessionFactory> factories = ImmutableMultimap.builder(); + kinds.forEach( + (imageName, caps) -> { + Map<String, String> imageProperties = getImageProperties(imageName); + + for (int i = 0; i < maxSessionCount; i++) { + factories.put( + caps, + new KubernetesSessionFactory( + tracer, + clientFactory, + options.getSessionTimeout(), + getServerStartTimeout(), + kubernetesClient, + namespace, + imageName, + caps, + imageProperties, + videoImage, + assetsPath, + capabilities -> options.getSlotMatcher().matches(caps, capabilities), + resourceLimits, + resourceRequests, + labels, + annotations)); + } + LOG.info( + String.format( + "Mapping %s to Kubernetes image %s %d times", caps, imageName, maxSessionCount)); + }); + return factories.build().asMap(); + } + + private String getVideoImage() { + String videoImage = config.get(KUBERNETES_SECTION, "video-image").orElse(DEFAULT_VIDEO_IMAGE); + if (videoImage.equalsIgnoreCase("false")) { + return null; + } + return videoImage; + } + + private KubernetesAssetsPath getAssetsPath() { + Optional<String> assetsPath = config.get(KUBERNETES_SECTION, "assets-path"); + // In Kubernetes, host path and pod path are typically the same for mounted volumes + return assetsPath.map(path -> new KubernetesAssetsPath(path, path)).orElse(null); + } + + private Map<String, String> getResourceLimits() { + Map<String, String> limits = new HashMap<>(); + config.get(KUBERNETES_SECTION, "cpu-limit").ifPresent(cpu -> limits.put("cpu", cpu)); + config.get(KUBERNETES_SECTION, "memory-limit").ifPresent(mem -> limits.put("memory", mem)); + return limits; + } + + private Map<String, String> getResourceRequests() { + Map<String, String> requests = new HashMap<>(); + config.get(KUBERNETES_SECTION, "cpu-request").ifPresent(cpu -> requests.put("cpu", cpu)); + config.get(KUBERNETES_SECTION, "memory-request").ifPresent(mem -> requests.put("memory", mem)); + return requests; + } + + private Map<String, String> getLabels() { + List<String> labelList = config.getAll(KUBERNETES_SECTION, "labels").orElseGet(Collections::emptyList); + Map<String, String> labels = new HashMap<>(); + + for (int i = 0; i < labelList.size(); i += 2) { + if (i + 1 < labelList.size()) { + labels.put(labelList.get(i), labelList.get(i + 1)); + } + } + return labels; + } + + private Map<String, String> getAnnotations() { + List<String> annotationList = config.getAll(KUBERNETES_SECTION, "annotations").orElseGet(Collections::emptyList); + Map<String, String> annotations = new HashMap<>(); + + for (int i = 0; i < annotationList.size(); i += 2) { + if (i + 1 < annotationList.size()) { + annotations.put(annotationList.get(i), annotationList.get(i + 1)); + } + } + return annotations; + } + + private Map<String, String> getImageProperties(String imageName) { + // Store any image-specific configuration + Map<String, String> properties = new HashMap<>(); + properties.put("imageName", imageName); + + // Add image pull policy if configured + config.get(KUBERNETES_SECTION, "image-pull-policy") + .ifPresent(policy -> properties.put("imagePullPolicy", policy)); + + return properties; + } +}
diff --git a/java/src/org/openqa/selenium/grid/node/k8s/KubernetesSession.java b/java/src/org/openqa/selenium/grid/node/k8s/KubernetesSession.java new file mode 100644 index 0000000..4a24ce4 --- /dev/null +++ b/java/src/org/openqa/selenium/grid/node/k8s/KubernetesSession.java
@@ -0,0 +1,127 @@ +// Licensed to the Software Freedom Conservancy (SFC) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The SFC licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.openqa.selenium.grid.node.k8s; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.BatchV1Api; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1Job; +import io.kubernetes.client.openapi.models.V1Pod; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.openqa.selenium.Capabilities; +import org.openqa.selenium.grid.node.DefaultActiveSession; +import org.openqa.selenium.internal.Require; +import org.openqa.selenium.remote.Dialect; +import org.openqa.selenium.remote.SessionId; +import org.openqa.selenium.remote.http.HttpClient; +import org.openqa.selenium.remote.tracing.Tracer; + +public class KubernetesSession extends DefaultActiveSession { + + private static final Logger LOG = Logger.getLogger(KubernetesSession.class.getName()); + private final ApiClient kubernetesClient; + private final String namespace; + private final String jobName; + private final String podName; + private final String videoPodName; + private final KubernetesAssetsPath assetsPath; + + KubernetesSession( + ApiClient kubernetesClient, + String namespace, + String jobName, + String podName, + String videoPodName, + Tracer tracer, + HttpClient client, + SessionId id, + URL url, + Capabilities stereotype, + Capabilities capabilities, + Dialect downstream, + Dialect upstream, + Instant startTime, + KubernetesAssetsPath assetsPath) { + super(tracer, client, id, url, downstream, upstream, stereotype, capabilities, startTime); + this.kubernetesClient = Require.nonNull("Kubernetes client", kubernetesClient); + this.namespace = Require.nonNull("Namespace", namespace); + this.jobName = Require.nonNull("Job name", jobName); + this.podName = Require.nonNull("Pod name", podName); + this.videoPodName = videoPodName; + this.assetsPath = Require.nonNull("Assets path", assetsPath); + } + + @Override + public void stop() { + if (videoPodName != null) { + deleteVideoPod(); + } + saveLogs(); + deleteJob(); + super.stop(); + } + + private void deleteVideoPod() { + try { + CoreV1Api coreApi = new CoreV1Api(kubernetesClient); + coreApi.deleteNamespacedPod(videoPodName, namespace); + LOG.info(String.format("Deleted video pod: %s", videoPodName)); + } catch (Exception e) { + LOG.log( + Level.WARNING, + String.format("Error deleting video pod %s: %s", videoPodName, e.getMessage()), + e); + } + } + + private void saveLogs() { + String sessionAssetsPath = assetsPath.getHostPath(getId()); + String seleniumServerLog = String.format("%s/selenium-server.log", sessionAssetsPath); + try { + CoreV1Api coreApi = new CoreV1Api(kubernetesClient); + String logs = coreApi.readNamespacedPodLog(podName, namespace).execute(); + if (logs != null) { + List<String> logLines = List.of(logs.split("\n")); + Files.write(Paths.get(seleniumServerLog), logLines); + } + } catch (Exception e) { + LOG.log(Level.WARNING, "Error saving logs", e); + } + } + + private void deleteJob() { + try { + BatchV1Api batchApi = new BatchV1Api(kubernetesClient); + batchApi.deleteNamespacedJob(jobName, namespace); + LOG.info(String.format("Deleted Kubernetes job: %s", jobName)); + } catch (Exception e) { + LOG.log( + Level.WARNING, + String.format("Error deleting job %s: %s", jobName, e.getMessage()), + e); + } + } +}
diff --git a/java/src/org/openqa/selenium/grid/node/k8s/KubernetesSessionFactory.java b/java/src/org/openqa/selenium/grid/node/k8s/KubernetesSessionFactory.java new file mode 100644 index 0000000..11a0159 --- /dev/null +++ b/java/src/org/openqa/selenium/grid/node/k8s/KubernetesSessionFactory.java
@@ -0,0 +1,526 @@ +// Licensed to the Software Freedom Conservancy (SFC) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The SFC licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.openqa.selenium.grid.node.k8s; + +import static java.util.Optional.ofNullable; +import static org.openqa.selenium.remote.Dialect.W3C; +import static org.openqa.selenium.remote.http.Contents.string; +import static org.openqa.selenium.remote.http.HttpMethod.GET; +import static org.openqa.selenium.remote.tracing.Tags.EXCEPTION; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.BatchV1Api; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.*; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Duration; +import java.time.Instant; +import java.util.*; +import java.util.function.Predicate; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.openqa.selenium.Capabilities; +import org.openqa.selenium.ImmutableCapabilities; +import org.openqa.selenium.PersistentCapabilities; +import org.openqa.selenium.RetrySessionRequestException; +import org.openqa.selenium.SessionNotCreatedException; +import org.openqa.selenium.TimeoutException; +import org.openqa.selenium.WebDriverException; +import org.openqa.selenium.grid.data.CreateSessionRequest; +import org.openqa.selenium.grid.node.ActiveSession; +import org.openqa.selenium.grid.node.SessionFactory; +import org.openqa.selenium.internal.Either; +import org.openqa.selenium.internal.Require; +import org.openqa.selenium.json.Json; +import org.openqa.selenium.remote.Command; +import org.openqa.selenium.remote.Dialect; +import org.openqa.selenium.remote.DriverCommand; +import org.openqa.selenium.remote.ProtocolHandshake; +import org.openqa.selenium.remote.Response; +import org.openqa.selenium.remote.SessionId; +import org.openqa.selenium.remote.http.ClientConfig; +import org.openqa.selenium.remote.http.HttpClient; +import org.openqa.selenium.remote.http.HttpRequest; +import org.openqa.selenium.remote.http.HttpResponse; +import org.openqa.selenium.remote.tracing.AttributeKey; +import org.openqa.selenium.remote.tracing.AttributeMap; +import org.openqa.selenium.remote.tracing.Span; +import org.openqa.selenium.remote.tracing.Status; +import org.openqa.selenium.remote.tracing.Tracer; +import org.openqa.selenium.support.ui.FluentWait; +import org.openqa.selenium.support.ui.Wait; + +public class KubernetesSessionFactory implements SessionFactory { + + private static final Logger LOG = Logger.getLogger(KubernetesSessionFactory.class.getName()); + private static final Json JSON = new Json(); + + private final Tracer tracer; + private final HttpClient.Factory clientFactory; + private final Duration sessionTimeout; + private final Duration serverStartTimeout; + private final ApiClient kubernetesClient; + private final String namespace; + private final String browserImage; + private final Capabilities stereotype; + private final Map<String, String> imageProperties; + private final String videoImage; + private final KubernetesAssetsPath assetsPath; + private final Predicate<Capabilities> predicate; + private final Map<String, String> resourceLimits; + private final Map<String, String> resourceRequests; + private final Map<String, String> labels; + private final Map<String, String> annotations; + + public KubernetesSessionFactory( + Tracer tracer, + HttpClient.Factory clientFactory, + Duration sessionTimeout, + Duration serverStartTimeout, + ApiClient kubernetesClient, + String namespace, + String browserImage, + Capabilities stereotype, + Map<String, String> imageProperties, + String videoImage, + KubernetesAssetsPath assetsPath, + Predicate<Capabilities> predicate, + Map<String, String> resourceLimits, + Map<String, String> resourceRequests, + Map<String, String> labels, + Map<String, String> annotations) { + this.tracer = Require.nonNull("Tracer", tracer); + this.clientFactory = Require.nonNull("HTTP client", clientFactory); + this.sessionTimeout = Require.nonNull("Session timeout", sessionTimeout); + this.serverStartTimeout = Require.nonNull("Server start timeout", serverStartTimeout); + this.kubernetesClient = Require.nonNull("Kubernetes client", kubernetesClient); + this.namespace = Require.nonNull("Namespace", namespace); + this.browserImage = Require.nonNull("Browser image", browserImage); + this.stereotype = ImmutableCapabilities.copyOf(Require.nonNull("Stereotype", stereotype)); + this.imageProperties = Require.nonNull("Image properties", imageProperties); + this.videoImage = videoImage; + this.assetsPath = assetsPath; + this.predicate = Require.nonNull("Accepted capabilities predicate", predicate); + this.resourceLimits = Require.nonNull("Resource limits", resourceLimits); + this.resourceRequests = Require.nonNull("Resource requests", resourceRequests); + this.labels = Require.nonNull("Labels", labels); + this.annotations = Require.nonNull("Annotations", annotations); + } + + @Override + public Capabilities getStereotype() { + return stereotype; + } + + @Override + public boolean test(Capabilities capabilities) { + return predicate.test(capabilities); + } + + @Override + public Either<WebDriverException, ActiveSession> apply(CreateSessionRequest sessionRequest) { + LOG.info("Starting Kubernetes session for " + sessionRequest.getDesiredCapabilities()); + + // Generate unique identifier for Job and Pod naming + String browserName = sessionRequest.getDesiredCapabilities().getBrowserName(); + if (browserName != null && !browserName.isEmpty()) { + browserName = browserName.toLowerCase(); + } else { + browserName = "unknown"; + } + long timestamp = System.currentTimeMillis(); + String uniqueId = UUID.randomUUID().toString().substring(0, 8); + String sessionIdentifier = String.format("%s-%d-%s", browserName, timestamp, uniqueId); + String jobName = "selenium-session-" + sessionIdentifier; + + try (Span span = tracer.getCurrentContext().createSpan("kubernetes_session_factory.apply")) { + AttributeMap attributeMap = tracer.createAttributeMap(); + attributeMap.put(AttributeKey.LOGGER_CLASS.getKey(), this.getClass().getName()); + + LOG.info("Creating Kubernetes Job: " + jobName); + + // Create the Job + V1Job job = createBrowserJob(jobName, sessionRequest.getDesiredCapabilities(), sessionIdentifier); + V1Job createdJob; + String podName; + + try { + BatchV1Api batchApi = new BatchV1Api(kubernetesClient); + createdJob = batchApi.createNamespacedJob(namespace, job).execute(); + LOG.info("Job created: " + createdJob.getMetadata().getName()); + + // Wait for pod to be created and get its name + podName = waitForPodCreation(jobName); + LOG.info("Pod created: " + podName); + + // Wait for pod to be running + waitForPodRunning(podName); + LOG.info("Pod is running: " + podName); + + // Get pod IP + String podIp = getPodIp(podName); + URL remoteAddress = new URL(String.format("http://%s:4444", podIp)); + + attributeMap.put("kubernetes.browser.image", browserImage); + attributeMap.put("kubernetes.job.name", jobName); + attributeMap.put("kubernetes.pod.name", podName); + attributeMap.put("kubernetes.pod.ip", podIp); + attributeMap.put("kubernetes.server.url", remoteAddress.toString()); + + // Create HTTP client + ClientConfig clientConfig = + ClientConfig.defaultConfig().baseUrl(remoteAddress).readTimeout(sessionTimeout); + HttpClient client = clientFactory.createClient(clientConfig); + + LOG.info(String.format("Waiting for server to start (pod: %s, url %s)", podName, remoteAddress)); + try { + waitForServerToStart(client, serverStartTimeout); + } catch (TimeoutException e) { + span.setAttribute(AttributeKey.ERROR.getKey(), true); + span.setStatus(Status.CANCELLED); + EXCEPTION.accept(attributeMap, e); + attributeMap.put( + AttributeKey.EXCEPTION_MESSAGE.getKey(), + "Unable to connect to Kubernetes pod. Deleting job: " + e.getMessage()); + span.addEvent(AttributeKey.EXCEPTION_EVENT.getKey(), attributeMap); + + deleteJob(jobName); + String message = String.format("Unable to connect to Kubernetes pod (job: %s)", jobName); + LOG.warning(message); + client.close(); + return Either.left(new RetrySessionRequestException(message)); + } + LOG.info(String.format("Server is ready (pod: %s)", podName)); + + // Create session + Command command = + new Command(null, DriverCommand.NEW_SESSION(sessionRequest.getDesiredCapabilities())); + ProtocolHandshake.Result result; + Response response; + try { + result = new ProtocolHandshake().createSession(client, command); + response = result.createResponse(); + attributeMap.put(AttributeKey.DRIVER_RESPONSE.getKey(), response.toString()); + } catch (IOException | RuntimeException e) { + span.setAttribute(AttributeKey.ERROR.getKey(), true); + span.setStatus(Status.CANCELLED); + EXCEPTION.accept(attributeMap, e); + attributeMap.put( + AttributeKey.EXCEPTION_MESSAGE.getKey(), + "Unable to create session. Deleting job: " + e.getMessage()); + span.addEvent(AttributeKey.EXCEPTION_EVENT.getKey(), attributeMap); + + deleteJob(jobName); + String message = "Unable to create session: " + e.getMessage(); + LOG.log(Level.WARNING, message, e); + client.close(); + return Either.left(new SessionNotCreatedException(message)); + } + + SessionId id = new SessionId(response.getSessionId()); + Capabilities capabilities = new ImmutableCapabilities((Map<?, ?>) response.getValue()); + Capabilities mergedCapabilities = sessionRequest.getDesiredCapabilities().merge(capabilities); + mergedCapabilities = addForwardCdpEndpoint(mergedCapabilities, podIp, id.toString()); + + // Handle video recording pod + String videoPodName = null; + Optional<KubernetesAssetsPath> path = ofNullable(this.assetsPath); + if (path.isPresent()) { + String podPath = path.get().getPodPath(id); + saveSessionCapabilities(mergedCapabilities, podPath); + String hostPath = path.get().getHostPath(id); + videoPodName = startVideoPod(mergedCapabilities, podIp, hostPath, sessionIdentifier); + } + + Dialect downstream = + sessionRequest.getDownstreamDialects().contains(result.getDialect()) + ? result.getDialect() + : W3C; + attributeMap.put(AttributeKey.DOWNSTREAM_DIALECT.getKey(), downstream.toString()); + attributeMap.put(AttributeKey.DRIVER_RESPONSE.getKey(), response.toString()); + + span.addEvent("Kubernetes driver service created session", attributeMap); + LOG.fine( + String.format( + "Created session: %s - %s (job: %s, pod: %s)", + id, mergedCapabilities, jobName, podName)); + + return Either.right( + new KubernetesSession( + kubernetesClient, + namespace, + jobName, + podName, + videoPodName, + tracer, + client, + id, + remoteAddress, + stereotype, + mergedCapabilities, + downstream, + result.getDialect(), + Instant.now(), + assetsPath)); + + } catch (Exception e) { + String message = "Failed to create Kubernetes Job: " + e.getMessage(); + LOG.log(Level.SEVERE, message, e); + return Either.left(new SessionNotCreatedException(message)); + } + } catch (Exception e) { + String message = "Unexpected error creating Kubernetes session: " + e.getMessage(); + LOG.log(Level.SEVERE, message, e); + return Either.left(new SessionNotCreatedException(message)); + } + } + + private V1Job createBrowserJob(String jobName, Capabilities capabilities, String sessionIdentifier) { + // Build environment variables + List<V1EnvVar> envVars = new ArrayList<>(); + envVars.add(new V1EnvVar().name("SE_NODE_MAX_SESSIONS").value("1")); + envVars.add(new V1EnvVar().name("SE_NODE_SESSION_TIMEOUT").value(String.valueOf(sessionTimeout.getSeconds()))); + + // Add screen resolution if specified + Object screenResolution = capabilities.getCapability("se:screenResolution"); + if (screenResolution != null) { + envVars.add(new V1EnvVar().name("SE_SCREEN_WIDTH").value(screenResolution.toString().split("x")[0])); + envVars.add(new V1EnvVar().name("SE_SCREEN_HEIGHT").value(screenResolution.toString().split("x")[1])); + } + + // Add timezone if specified + Object timeZone = capabilities.getCapability("se:timeZone"); + if (timeZone != null) { + envVars.add(new V1EnvVar().name("TZ").value(timeZone.toString())); + } + + // Build resource requirements + V1ResourceRequirements resources = new V1ResourceRequirements(); + if (!resourceRequests.isEmpty()) { + Map<String, io.kubernetes.client.custom.Quantity> requests = new HashMap<>(); + resourceRequests.forEach((k, v) -> requests.put(k, io.kubernetes.client.custom.Quantity.fromString(v))); + resources.setRequests(requests); + } + if (!resourceLimits.isEmpty()) { + Map<String, io.kubernetes.client.custom.Quantity> limits = new HashMap<>(); + resourceLimits.forEach((k, v) -> limits.put(k, io.kubernetes.client.custom.Quantity.fromString(v))); + resources.setLimits(limits); + } + + // Build container + V1Container container = new V1Container() + .name("selenium-browser") + .image(browserImage) + .env(envVars) + .resources(resources) + .addPortsItem(new V1ContainerPort().containerPort(4444).protocol("TCP")); + + // Build pod spec + V1PodSpec podSpec = new V1PodSpec() + .restartPolicy("Never") + .addContainersItem(container); + + // Build pod template + Map<String, String> podLabels = new HashMap<>(labels); + podLabels.put("selenium-session", sessionIdentifier); + podLabels.put("app", "selenium"); + podLabels.put("component", "browser"); + + V1PodTemplateSpec podTemplate = new V1PodTemplateSpec() + .metadata(new V1ObjectMeta() + .labels(podLabels) + .annotations(annotations)) + .spec(podSpec); + + // Build job spec + V1JobSpec jobSpec = new V1JobSpec() + .template(podTemplate) + .backoffLimit(0) // Don't retry failed jobs + .ttlSecondsAfterFinished(300); // Clean up after 5 minutes + + // Build job + Map<String, String> jobLabels = new HashMap<>(labels); + jobLabels.put("selenium-session", sessionIdentifier); + jobLabels.put("app", "selenium"); + + return new V1Job() + .apiVersion("batch/v1") + .kind("Job") + .metadata(new V1ObjectMeta() + .name(jobName) + .namespace(namespace) + .labels(jobLabels) + .annotations(annotations)) + .spec(jobSpec); + } + + private String waitForPodCreation(String jobName) throws TimeoutException { + CoreV1Api coreApi = new CoreV1Api(kubernetesClient); + String labelSelector = "job-name=" + jobName; + + Wait<CoreV1Api> wait = new FluentWait<>(coreApi) + .withTimeout(Duration.ofSeconds(30)) + .pollingEvery(Duration.ofSeconds(1)) + .ignoring(Exception.class); + + try { + return wait.until(api -> { + try { + V1PodList podList = api.listNamespacedPod(namespace).execute(); + if (podList.getItems() != null && !podList.getItems().isEmpty()) { + for (V1Pod pod : podList.getItems()) { + if (pod.getMetadata().getLabels() != null && + jobName.equals(pod.getMetadata().getLabels().get("job-name"))) { + return pod.getMetadata().getName(); + } + } + } + } catch (Exception e) { + LOG.log(Level.FINE, "Waiting for pod creation", e); + } + return null; + }); + } catch (org.openqa.selenium.TimeoutException e) { + throw new TimeoutException("Timeout waiting for pod creation for job: " + jobName); + } + } + + private void waitForPodRunning(String podName) throws TimeoutException { + CoreV1Api coreApi = new CoreV1Api(kubernetesClient); + + Wait<CoreV1Api> wait = new FluentWait<>(coreApi) + .withTimeout(Duration.ofSeconds(60)) + .pollingEvery(Duration.ofSeconds(2)) + .ignoring(Exception.class); + + try { + wait.until(api -> { + try { + V1Pod pod = api.readNamespacedPod(podName, namespace).execute(); + String phase = pod.getStatus().getPhase(); + return "Running".equals(phase); + } catch (Exception e) { + LOG.log(Level.FINE, "Waiting for pod to be running", e); + } + return false; + }); + } catch (org.openqa.selenium.TimeoutException e) { + throw new TimeoutException("Timeout waiting for pod to be running: " + podName); + } + } + + private String getPodIp(String podName) throws Exception { + CoreV1Api coreApi = new CoreV1Api(kubernetesClient); + V1Pod pod = coreApi.readNamespacedPod(podName, namespace).execute(); + return pod.getStatus().getPodIP(); + } + + private void waitForServerToStart(HttpClient client, Duration timeout) { + HttpRequest request = new HttpRequest(GET, "/status"); + Wait<HttpClient> wait = new FluentWait<>(client) + .withTimeout(timeout) + .pollingEvery(Duration.ofMillis(500)) + .ignoring(RuntimeException.class); + + wait.until(c -> { + HttpResponse response = c.execute(request); + LOG.fine(string(response)); + return 200 == response.getStatus(); + }); + } + + private Capabilities addForwardCdpEndpoint( + Capabilities sessionCapabilities, String podIp, String sessionId) { + String cdpPath = String.format("/session/%s/se/cdp", sessionId); + return new PersistentCapabilities(sessionCapabilities) + .setCapability("se:cdp", URI.create(String.format("ws://%s:4444%s", podIp, cdpPath))); + } + + private void saveSessionCapabilities(Capabilities capabilities, String path) { + try { + Files.createDirectories(Paths.get(path)); + String capsJson = JSON.toJson(capabilities); + Files.write(Paths.get(path, "capabilities.json"), capsJson.getBytes(Charset.defaultCharset())); + } catch (IOException e) { + LOG.log(Level.WARNING, "Failed to save session capabilities", e); + } + } + + private String startVideoPod(Capabilities capabilities, String podIp, String hostPath, String sessionIdentifier) { + if (videoImage == null || "false".equals(videoImage)) { + return null; + } + + Boolean recordVideo = (Boolean) capabilities.getCapability("se:recordVideo"); + if (recordVideo == null || !recordVideo) { + return null; + } + + try { + String videoPodName = "selenium-video-" + sessionIdentifier; + + V1Container videoContainer = new V1Container() + .name("video-recorder") + .image(videoImage) + .addEnvItem(new V1EnvVar().name("DISPLAY_CONTAINER_NAME").value(podIp)) + .addEnvItem(new V1EnvVar().name("FILE_NAME").value(sessionIdentifier + ".mp4")); + + V1PodSpec podSpec = new V1PodSpec() + .restartPolicy("Never") + .addContainersItem(videoContainer); + + Map<String, String> podLabels = new HashMap<>(labels); + podLabels.put("selenium-session", sessionIdentifier); + podLabels.put("app", "selenium"); + podLabels.put("component", "video"); + + V1Pod videoPod = new V1Pod() + .apiVersion("v1") + .kind("Pod") + .metadata(new V1ObjectMeta() + .name(videoPodName) + .namespace(namespace) + .labels(podLabels)) + .spec(podSpec); + + CoreV1Api coreApi = new CoreV1Api(kubernetesClient); + coreApi.createNamespacedPod(namespace, videoPod); + LOG.info("Video recording pod created: " + videoPodName); + return videoPodName; + } catch (Exception e) { + LOG.log(Level.WARNING, "Failed to create video recording pod", e); + return null; + } + } + + private void deleteJob(String jobName) { + try { + BatchV1Api batchApi = new BatchV1Api(kubernetesClient); + batchApi.deleteNamespacedJob(jobName, namespace); + } catch (Exception e) { + LOG.log(Level.WARNING, "Failed to delete job: " + jobName, e); + } + } +}
diff --git a/java/src/org/openqa/selenium/grid/node/k8s/README.md b/java/src/org/openqa/selenium/grid/node/k8s/README.md new file mode 100644 index 0000000..484dfda --- /dev/null +++ b/java/src/org/openqa/selenium/grid/node/k8s/README.md
@@ -0,0 +1,337 @@ +# Kubernetes Node for Selenium Grid + +This implementation provides a Kubernetes-based Node for Selenium Grid that creates Kubernetes Jobs for each browser session, similar to how the Docker Node creates containers. + +## Overview + +The Kubernetes Node implementation consists of the following components: + +- **KubernetesSession**: Manages the lifecycle of a browser session running in a Kubernetes Pod +- **KubernetesSessionFactory**: Creates Kubernetes Jobs and establishes connections to browser pods +- **KubernetesOptions**: Parses and manages configuration from config files or CLI flags +- **KubernetesFlags**: Defines command-line flags for Kubernetes configuration +- **KubernetesAssetsPath**: Manages paths for session assets (logs, videos, etc.) + +## Architecture + +### How It Works + +1. **Session Request**: When a new session is requested, the `KubernetesSessionFactory` is invoked +2. **Job Creation**: A Kubernetes Job is created with a Pod spec containing the browser container +3. **Pod Startup**: The factory waits for the Pod to be created and become running +4. **Connection**: Once running, an HTTP client connects to the Pod's IP address +5. **Session Creation**: A WebDriver session is created via the protocol handshake +6. **Session Management**: The `KubernetesSession` manages the active session +7. **Cleanup**: When the session ends, the Job (and its Pod) are deleted + +### Comparison with Docker Node + +| Feature | Docker Node | Kubernetes Node | +|---------|-------------|-----------------| +| Resource Type | Container | Job → Pod | +| Client Library | Custom Docker client | Official Kubernetes Java client | +| Networking | Docker networks | Kubernetes Pod networking | +| Storage | Volume mounts | PersistentVolumeClaims/EmptyDir | +| Resource Limits | Docker host config | Kubernetes resources | +| Cleanup | Container stop/remove | Job deletion (cascades to Pod) | + +## Configuration + +### Using Configuration File (TOML) + +```toml +[k8s] +namespace = "selenium" +server-start-timeout = 60 + +configs = [ + "selenium/standalone-chrome:latest", "{\"browserName\": \"chrome\"}", + "selenium/standalone-firefox:latest", "{\"browserName\": \"firefox\"}" +] + +cpu-request = "500m" +memory-request = "1Gi" +cpu-limit = "1" +memory-limit = "2Gi" + +labels = ["environment", "production"] +video-image = "selenium/video:latest" +assets-path = "/opt/selenium/assets" +``` + +### Using CLI Flags + +```bash +java -jar selenium-server.jar node \ + --k8s-namespace selenium \ + --k8s selenium/standalone-chrome:latest '{"browserName": "chrome"}' \ + --k8s selenium/standalone-firefox:latest '{"browserName": "firefox"}' \ + --k8s-cpu-request 500m \ + --k8s-memory-request 1Gi \ + --k8s-cpu-limit 1 \ + --k8s-memory-limit 2Gi \ + --k8s-labels environment production team qa \ + --k8s-video-image selenium/video:latest \ + --k8s-assets-path /opt/selenium/assets +``` + +### Configuration Options + +| Option | CLI Flag | Config | Description | Default | +|--------|----------|--------|-------------|---------| +| Kubeconfig Path | `--k8s-kubeconfig` | `k8s.kubeconfig` | Path to kubeconfig file | Default kubeconfig or in-cluster | +| Namespace | `--k8s-namespace` | `k8s.namespace` | Kubernetes namespace | `default` | +| Server Start Timeout | `--k8s-server-start-timeout` | `k8s.server-start-timeout` | Pod startup timeout (seconds) | `60` | +| Browser Configs | `--k8s` or `-K` | `k8s.configs` | Image to capabilities mapping | - | +| CPU Request | `--k8s-cpu-request` | `k8s.cpu-request` | CPU resource request | - | +| Memory Request | `--k8s-memory-request` | `k8s.memory-request` | Memory resource request | - | +| CPU Limit | `--k8s-cpu-limit` | `k8s.cpu-limit` | CPU resource limit | - | +| Memory Limit | `--k8s-memory-limit` | `k8s.memory-limit` | Memory resource limit | - | +| Labels | `--k8s-labels` | `k8s.labels` | Custom Pod labels (key-value pairs) | - | +| Annotations | `--k8s-annotations` | `k8s.annotations` | Custom Pod annotations | - | +| Video Image | `--k8s-video-image` | `k8s.video-image` | Video recording image | `false` (disabled) | +| Assets Path | `--k8s-assets-path` | `k8s.assets-path` | Path for session assets | `/opt/selenium/assets` | +| Image Pull Policy | `--k8s-image-pull-policy` | `k8s.image-pull-policy` | Image pull policy | - | + +## Prerequisites + +### Kubernetes Cluster + +You need access to a Kubernetes cluster with: +- Kubernetes 1.20+ (recommended) +- Sufficient resources for browser pods +- Network connectivity between Grid Node and Pods + +### Authentication + +The Node can authenticate to Kubernetes using: +1. **In-cluster configuration**: When running inside a Kubernetes cluster with a ServiceAccount +2. **Kubeconfig file**: Specify `--k8s-kubeconfig /path/to/config` +3. **Default kubeconfig**: Uses `~/.kube/config` by default + +### RBAC Permissions + +The ServiceAccount (or kubeconfig user) needs these permissions: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: selenium-grid-node + namespace: selenium +rules: +- apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "delete", "get", "list"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["create", "delete", "get", "list", "watch"] +- apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] +``` + +## Deployment Examples + +### Standalone Node + +Deploy the Grid Node as a Kubernetes Deployment: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: selenium-grid-k8s-node + namespace: selenium +spec: + replicas: 1 + selector: + matchLabels: + app: selenium-grid + component: k8s-node + template: + metadata: + labels: + app: selenium-grid + component: k8s-node + spec: + serviceAccountName: selenium-grid-node + containers: + - name: node + image: selenium/node-k8s:latest + ports: + - containerPort: 5555 + env: + - name: SE_EVENT_BUS_HOST + value: "selenium-hub" + - name: SE_EVENT_BUS_PUBLISH_PORT + value: "4442" + - name: SE_EVENT_BUS_SUBSCRIBE_PORT + value: "4443" + volumeMounts: + - name: config + mountPath: /opt/selenium/config.toml + subPath: config.toml + - name: assets + mountPath: /opt/selenium/assets + volumes: + - name: config + configMap: + name: k8s-node-config + - name: assets + emptyDir: {} +``` + +### With Config Map + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: k8s-node-config + namespace: selenium +data: + config.toml: | + [k8s] + namespace = "selenium" + configs = [ + "selenium/standalone-chrome:latest", "{\"browserName\": \"chrome\"}", + "selenium/standalone-firefox:latest", "{\"browserName\": \"firefox\"}" + ] + cpu-request = "500m" + memory-request = "1Gi" + cpu-limit = "1" + memory-limit = "2Gi" +``` + +## Features + +### Resource Management + +The Kubernetes Node supports fine-grained resource control: + +```bash +--k8s-cpu-request 500m # Request 0.5 CPU cores +--k8s-memory-request 1Gi # Request 1 GB memory +--k8s-cpu-limit 2 # Limit to 2 CPU cores +--k8s-memory-limit 4Gi # Limit to 4 GB memory +``` + +### Custom Labels and Annotations + +Add custom metadata to browser Jobs/Pods: + +```bash +--k8s-labels environment production team qa managed-by selenium +--k8s-annotations prometheus.io/scrape true prometheus.io/port 4444 +``` + +### Video Recording + +Enable video recording for sessions: + +```bash +--k8s-video-image selenium/video:latest +``` + +The video recording pod will be automatically created alongside the browser pod and deleted when the session ends. + +### Session Assets + +Logs and other session assets are saved to the configured assets path: + +```bash +--k8s-assets-path /opt/selenium/assets +``` + +Each session gets its own directory: `/opt/selenium/assets/{session-id}/` + +## Troubleshooting + +### Pod Creation Timeout + +If pods fail to start within the timeout: +- Increase `--k8s-server-start-timeout` +- Check image pull time with `kubectl describe pod` +- Verify sufficient cluster resources + +### Permission Denied + +If you see `Forbidden` errors: +- Verify RBAC permissions are correctly configured +- Check ServiceAccount is bound to the Role +- Ensure namespace matches configuration + +### Network Connectivity + +If the Node cannot connect to Pods: +- Verify Pod networking is functional +- Check firewall rules +- Ensure Network Policies allow traffic + +### Logs + +View Node logs: +```bash +kubectl logs -f deployment/selenium-grid-k8s-node -n selenium +``` + +View browser Pod logs: +```bash +kubectl logs selenium-session-chrome-{timestamp}-{uuid} -n selenium +``` + +## Integration with LocalNodeFactory + +The Kubernetes Node is automatically registered with `LocalNodeFactory` if Kubernetes configurations are present: + +```java +if (config.getAll("k8s", "configs").isPresent()) { + new KubernetesOptions(config) + .getKubernetesSessionFactories(tracer, clientFactory, nodeOptions) + .forEach((caps, factories) -> factories.forEach(factory -> builder.add(caps, factory))); +} +``` + +## Differences from OneShotNode + +The existing `OneShotNode` is a simple implementation that: +- Runs a single local WebDriver session per pod +- Drains immediately after session creation +- Doesn't create Jobs - just runs the browser locally + +This new Kubernetes Node implementation: +- Creates Kubernetes Jobs for each session (like Docker Node creates containers) +- Supports multiple concurrent sessions +- Provides full session lifecycle management +- Integrates with the Grid like Docker Node does + +## Development + +### Dependencies + +The Kubernetes Node uses the official Kubernetes Java client: + +```xml +<dependency> + <groupId>io.kubernetes</groupId> + <artifactId>client-java</artifactId> + <version>24.0.0-legacy</version> +</dependency> +``` + +### Building + +```bash +bazel build //java/src/org/openqa/selenium/grid/node/k8s +``` + +### Testing + +```bash +bazel test //java/test/org/openqa/selenium/grid/node/k8s:all +``` + +## License + +Licensed under the Apache License, Version 2.0. See LICENSE file for details.
diff --git a/java/src/org/openqa/selenium/grid/node/k8s/k8s-node-config.toml b/java/src/org/openqa/selenium/grid/node/k8s/k8s-node-config.toml new file mode 100644 index 0000000..bae0fbd --- /dev/null +++ b/java/src/org/openqa/selenium/grid/node/k8s/k8s-node-config.toml
@@ -0,0 +1,76 @@ +# Kubernetes Node Configuration Example +# This configuration file demonstrates how to set up a Selenium Grid Node +# that creates Kubernetes Jobs for browser sessions + +[node] +# Maximum number of concurrent sessions +max-sessions = 5 + +# Session timeout in seconds +session-timeout = 300 + +# Heartbeat period in seconds +heartbeat-period = 60 + +# Enable CDP (Chrome DevTools Protocol) +enable-cdp = true + +# Enable BiDi (WebDriver BiDi) +enable-bidi = true + +[k8s] +# Kubernetes namespace to create Jobs/Pods in +namespace = "selenium" + +# Path to kubeconfig file (optional, will use default or in-cluster config if not specified) +# kubeconfig = "/home/user/.kube/config" + +# Server startup timeout in seconds +server-start-timeout = 60 + +# Browser image configurations +# Format: image-name capability-json +configs = [ + "selenium/standalone-chrome:latest", "{\"browserName\": \"chrome\"}", + "selenium/standalone-firefox:latest", "{\"browserName\": \"firefox\"}", + "selenium/standalone-edge:latest", "{\"browserName\": \"MicrosoftEdge\"}" +] + +# Resource requests for browser pods +cpu-request = "500m" +memory-request = "1Gi" + +# Resource limits for browser pods +cpu-limit = "1" +memory-limit = "2Gi" + +# Custom labels for browser Jobs/Pods +# Format: key value pairs +labels = [ + "environment", "production", + "team", "qa", + "managed-by", "selenium-grid" +] + +# Custom annotations for browser Jobs/Pods +annotations = [ + "prometheus.io/scrape", "true", + "prometheus.io/port", "4444" +] + +# Video recording image (set to "false" to disable) +video-image = "selenium/video:latest" + +# Assets path for storing logs and videos +assets-path = "/opt/selenium/assets" + +# Image pull policy (Always, IfNotPresent, Never) +image-pull-policy = "IfNotPresent" + +[server] +# Grid Node server port +port = 5555 + +[network] +# Hostname or IP address of this node +# hostname = "node-hostname"
diff --git a/java/src/org/openqa/selenium/grid/node/local/BUILD.bazel b/java/src/org/openqa/selenium/grid/node/local/BUILD.bazel index 3a29f82..7452221 100644 --- a/java/src/org/openqa/selenium/grid/node/local/BUILD.bazel +++ b/java/src/org/openqa/selenium/grid/node/local/BUILD.bazel
@@ -22,6 +22,7 @@ "//java/src/org/openqa/selenium/grid/node", "//java/src/org/openqa/selenium/grid/node/config", "//java/src/org/openqa/selenium/grid/node/docker", + "//java/src/org/openqa/selenium/grid/node/k8s", "//java/src/org/openqa/selenium/grid/node/relay", "//java/src/org/openqa/selenium/grid/security", "//java/src/org/openqa/selenium/grid/server",
diff --git a/java/src/org/openqa/selenium/grid/node/local/LocalNodeFactory.java b/java/src/org/openqa/selenium/grid/node/local/LocalNodeFactory.java index 600f516..e3fb320 100644 --- a/java/src/org/openqa/selenium/grid/node/local/LocalNodeFactory.java +++ b/java/src/org/openqa/selenium/grid/node/local/LocalNodeFactory.java
@@ -34,6 +34,7 @@ import org.openqa.selenium.grid.node.config.DriverServiceSessionFactory; import org.openqa.selenium.grid.node.config.NodeOptions; import org.openqa.selenium.grid.node.docker.DockerOptions; +import org.openqa.selenium.grid.node.k8s.KubernetesOptions; import org.openqa.selenium.grid.node.relay.RelayOptions; import org.openqa.selenium.grid.security.SecretOptions; import org.openqa.selenium.grid.server.BaseServerOptions; @@ -94,6 +95,12 @@ public static Node create(Config config) { .forEach((caps, factories) -> factories.forEach(factory -> builder.add(caps, factory))); } + if (config.getAll("k8s", "configs").isPresent()) { + new KubernetesOptions(config) + .getKubernetesSessionFactories(tracer, clientFactory, nodeOptions) + .forEach((caps, factories) -> factories.forEach(factory -> builder.add(caps, factory))); + } + if (config.getAll("relay", "configs").isPresent()) { new RelayOptions(config) .getSessionFactories(tracer, clientFactory, sessionTimeout)