Fabric runtime 1.3 uses java 11.
Here is the jar file installed on this environment
Here are all the java related libraries on this environment
I am on runtime 1.3 and it uses Scala 2.12.17
I have pasted both stderr and stdout above
stdout InMemoryCacheClient class found. Proceeding with token caching. ZookeeperCache class found. Proceeding with token caching. Statement0-invokeGenerateTridentContext: Total time taken 288 msec Statement0-saveTokens: Total time taken 3 msec Statement0-setSparkConfigs: Total time taken 13 msec Statement0-setDynamicAllocationSparkConfigs: Total time taken 6 msec Statement0-setLocalProperties: Total time taken 0 msec Statement0-setHadoopConfigs: Total time taken 0 msec Statement0 completed in 313 msec [Python] Insert /synfs/nb_resource to sys.path. MARKED_DIR_DBFS: abfss://data_services_non_prod@onelake.dfs.fabric.microsoft.com/data_services_dev_products_lh.Lakehouse/Files/zingg/models/vendor_test_model/trainingData/marked/ UNMARKED_DIR_DBFS: abfss://data_services_non_prod@onelake.dfs.fabric.microsoft.com/data_services_dev_products_lh.Lakehouse/Files/zingg/models/vendor_test_model/trainingData/unmarked/ /opt/spark/python/lib/pyspark.zip/pyspark/sql/context.py:113: FutureWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead. <zingg.client.Arguments object at 0x7a8bc7e65cd0> set schema --------------------------------------------------------------------------- Py4JJavaError Traceback (most recent call last) Cell In[31], line 7 5 print(f"options: {options}") 6 print(f"zingg: {zingg}") ----> 7 zingg.initAndExecute() File ~/cluster-env/clonedenv/lib/python3.11/site-packages/zingg/client.py:280, in Zingg.initAndExecute(self) 278 self.client.execute() 279 else: --> 280 self.client.execute() File ~/cluster-env/clonedenv/lib/python3.11/site-packages/py4j/java_gateway.py:1304, in JavaMember.__call__(self, *args) 1298 command = proto.CALL_COMMAND_NAME +\ 1299 self.command_header +\ 1300 args_command +\ 1301 proto.END_COMMAND_PART 1303 answer = self.gateway_client.send_command(command) -> 1304 return_value = get_return_value( 1305 answer, self.gateway_client, self.target_id, self.name) 1307 for temp_arg in temp_args: 1308 temp_arg._detach() File /opt/spark/python/lib/pyspark.zip/pyspark/errors/exceptions/captured.py:179, in capture_sql_exception.<locals>.deco(*a, **kw) 177 def deco(*a: Any, **kw: Any) -> Any: 178 try: --> 179 return f(*a, **kw) 180 except Py4JJavaError as e: 181 converted = convert_exception(e.java_exception) File ~/cluster-env/clonedenv/lib/python3.11/site-packages/py4j/protocol.py:326, in get_return_value(answer, gateway_client, target_id, name) 324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client) 325 if answer[1] == REFERENCE_TYPE: --> 326 raise Py4JJavaError( 327 "An error occurred while calling {0}{1}{2}.\n". 328 format(target_id, ".", name), value) 329 else: 330 raise Py4JError( 331 "An error occurred while calling {0}{1}{2}. Trace:\n{3}\n". 332 format(target_id, ".", name, value)) Py4JJavaError: An error occurred while calling o6666.execute. : scala.NotImplementedError: an implementation is missing at scala.Predef$.$qmark$qmark$qmark(Predef.scala:288) at com.microsoft.fabric.spark.catalog.OnelakeExternalCatalog.functionExists(OnelakeExternalCatalog.scala:402) at com.microsoft.fabric.spark.catalog.InstrumentedExternalCatalog.$anonfun$functionExists$1(OnelakeExternalCatalog.scala:623) at scala.runtime.java8.JFunction0$mcZ$sp.apply(JFunction0$mcZ$sp.java:23) at com.microsoft.fabric.spark.catalog.metadata.Helpers$.timed(Helpers.scala:89) at com.microsoft.fabric.spark.catalog.InstrumentedExternalCatalog.functionExists(OnelakeExternalCatalog.scala:623) at org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener.$anonfun$functionExists$1(ExternalCatalogWithListener.scala:341) at scala.runtime.java8.JFunction0$mcZ$sp.apply(JFunction0$mcZ$sp.java:23) at com.microsoft.fabric.spark.catalog.metrics.Helpers$.timed(CatalogMetricsHelper.scala:64) at org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener.functionExists(ExternalCatalogWithListener.scala:341) at org.apache.spark.sql.catalyst.catalog.SessionCatalog.functionExists(SessionCatalog.scala:1566) at org.apache.spark.sql.internal.CatalogImpl.functionExists(CatalogImpl.scala:548) at zingg.spark.core.util.SparkFnRegistrar.registerUDF2(SparkFnRegistrar.java:21) at zingg.spark.core.preprocess.stopwords.SparkStopWordsRemover.registerUDF(SparkStopWordsRemover.java:60) at zingg.spark.core.preprocess.stopwords.SparkStopWordsRemover.init(SparkStopWordsRemover.java:65) at zingg.common.core.preprocess.IPreprocessors.executeAndBuildPreprocessedDF(IPreprocessors.java:53) at zingg.common.core.preprocess.IPreprocessors.preprocess(IPreprocessors.java:35) at zingg.common.core.executor.TrainingDataFinder.getPositiveSamples(TrainingDataFinder.java:198) at zingg.common.core.executor.TrainingDataFinder.execute(TrainingDataFinder.java:69) at zingg.common.client.Client.execute(Client.java:281) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) at py4j.Gateway.invoke(Gateway.java:282) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.GatewayConnection.run(GatewayConnection.java:238) at java.base/java.lang.Thread.run(Thread.java:829)
Will greatly appreciate if you can provide further direction.
I have sent the executor log and the code. Please let me know if any other information is required.
BELOW IS THE EXECUTOR LOG [{'containerId': 'container_1759340256258_0001_01_000001', 'nodeId': 'vm-2c612797:14554', 'containerLogMeta': [{'fileName': 'directory.info', 'length': 8630, 'lastModified': '2025-10-01T17:37:54.000GMT', 'metaData': {}}, {'fileName': 'launch_container.sh', 'length': 7535, 'lastModified': '2025-10-01T17:37:54.000GMT', 'metaData': {}}, {'fileName': 'prelaunch.err', 'length': 0, 'lastModified': '2025-10-01T17:37:54.000GMT', 'metaData': {}}, {'fileName': 'prelaunch.out', 'length': 100, 'lastModified': '2025-10-01T17:37:54.000GMT', 'metaData': {}}, {'fileName': 'stderr', 'length': 551052, 'lastModified': '2025-10-01T18:19:15.000GMT', 'metaData': {}}, {'fileName': 'stderr-2025-10-01-17', 'length': 497936, 'lastModified': '2025-10-01T17:59:40.000GMT', 'metaData': {}}, {'fileName': 'stderr-active', 'length': 53145, 'lastModified': '2025-10-01T18:19:15.000GMT', 'metaData': {}}, {'fileName': 'stdout', 'length': 11267, 'lastModified': '2025-10-01T18:17:02.000GMT', 'metaData': {}}, {'fileName': 'stdout-2025-10-01-17', 'length': 4862, 'lastModified': '2025-10-01T17:59:45.000GMT', 'metaData': {}}, {'fileName': 'stdout-active', 'length': 6351, 'lastModified': '2025-10-01T18:17:02.000GMT', 'metaData': {}}]}, {'containerId': 'container_1759340256258_0001_01_000003', 'nodeId': 'vm-4ed06444:14554', 'containerLogMeta': [{'fileName': 'directory.info', 'length': 8630, 'lastModified': '2025-10-01T17:53:44.000GMT', 'metaData': {}}, {'fileName': 'launch_container.sh', 'length': 7000, 'lastModified': '2025-10-01T17:53:44.000GMT', 'metaData': {}}, {'fileName': 'prelaunch.err', 'length': 0, 'lastModified': '2025-10-01T17:53:44.000GMT', 'metaData': {}}, {'fileName': 'prelaunch.out', 'length': 100, 'lastModified': '2025-10-01T17:53:44.000GMT', 'metaData': {}}, {'fileName': 'stderr', 'length': 37500, 'lastModified': '2025-10-01T18:18:55.000GMT', 'metaData': {}}, {'fileName': 'stderr-2025-10-01-17-1', 'length': 30512, 'lastModified': '2025-10-01T17:59:55.000GMT', 'metaData': {}}, {'fileName': 'stderr-active', 'length': 5225, 'lastModified': '2025-10-01T18:18:55.000GMT', 'metaData': {}}, {'fileName': 'stdout', 'length': 508, 'lastModified': '2025-10-01T17:53:47.000GMT', 'metaData': {}}, {'fileName': 'stdout-active', 'length': 0, 'lastModified': '2025-10-01T17:53:48.000GMT', 'metaData': {}}]}]
