Deinitialization support.

Author: Kerilk

include/cl_khr_icd2.h

@@ -18,6 +18,19 @@
 
 #include <CL/cl.h>
 
+#if !defined(CL_PLATFORM_UNLOADABLE_KHR)
+#define CL_PLATFORM_UNLOADABLE_KHR 0x0921
+#endif
+
+#if defined(CL_ENABLE_LAYERS) && !defined(CL_LAYER_API_VERSION_200)
+#define CL_LAYER_API_VERSION_200 200
+
+typedef cl_int CL_API_CALL
+clDeinitLayer_t(void);
+
+typedef clDeinitLayer_t *pfn_clDeinitLayer;
+#endif //defined(CL_ENABLE_LAYERS) && !defined(CL_LAYER_API_VERSION_200)
+
 #if !defined(CL_ICD2_TAG_KHR)
 #if INTPTR_MAX == INT32_MAX
 #define CL_ICD2_TAG_KHR ((intptr_t)0x434C3331)

loader/cllayerinfo.c

@@ -19,7 +19,6 @@
 #include "icd.h"
 #include <stdio.h>
 #include <stdlib.h>
-#include <CL/cl_layer.h>
 #if defined(_WIN32)
 #include <io.h>
 #include <share.h>
@@ -90,7 +89,7 @@ static void restore_outputs(void)
 void printLayerInfo(const struct KHRLayer *layer)
 {
     cl_layer_api_version api_version = 0;
-    pfn_clGetLayerInfo p_clGetLayerInfo = (pfn_clGetLayerInfo)(size_t)layer->p_clGetLayerInfo;
+    pfn_clGetLayerInfo p_clGetLayerInfo = layer->p_clGetLayerInfo;
     cl_int result = CL_SUCCESS;
     size_t sz;
 
@@ -113,20 +112,26 @@ void printLayerInfo(const struct KHRLayer *layer)
     }
 }
 
-int main (int argc, char *argv[])
+static void run_silently(void (*pfn)(void))
 {
-    (void)argc;
-    (void)argv;
     silence_layers();
     atexit(restore_outputs);
-    khrIcdInitialize();
+    pfn();
     restore_outputs();
     atexit(silence_layers);
+}
+
+int main (int argc, char *argv[])
+{
+    (void)argc;
+    (void)argv;
+    run_silently(&khrIcdInitialize);
     const struct KHRLayer *layer = khrFirstLayer;
     while (layer)
     {
         printLayerInfo(layer);
         layer = layer->next;
     }
+    run_silently(&khrIcdDeinitialize);
     return 0;
 }

loader/icd.c

@@ -19,14 +19,13 @@
 #include "icd.h"
 #include "icd_dispatch.h"
 #include "icd_envvars.h"
-#if defined(CL_ENABLE_LAYERS)
-#include <CL/cl_layer.h>
-#endif // defined(CL_ENABLE_LAYERS)
 #include <stdlib.h>
 #include <string.h>
 
 KHRicdVendor *khrIcdVendors = NULL;
+static KHRicdVendor *lastVendor = NULL;
 int khrEnableTrace = 0;
+static int khrDisableLibraryUnloading = 0;
 
 #if defined(CL_ENABLE_LAYERS)
 struct KHRLayer *khrFirstLayer = NULL;
@@ -43,6 +42,26 @@ void khrIcdInitializeTrace(void)
     {
         khrEnableTrace = 1;
     }
+    if (enableTrace)
+    {
+        khrIcd_free_getenv(enableTrace);
+    }
+}
+
+void khrIcdInitializeLibraryUnloading(void)
+{
+    char *disableLibraryUnloading = khrIcd_getenv("OCL_ICD_DISABLE_DYNAMIC_LIBRARY_UNLOADING");
+    if (disableLibraryUnloading && (strcmp(disableLibraryUnloading, "True") == 0 ||
+            strcmp(disableLibraryUnloading, "true") == 0 ||
+            strcmp(disableLibraryUnloading, "T") == 0 ||
+            strcmp(disableLibraryUnloading, "1") == 0))
+    {
+        khrDisableLibraryUnloading = 1;
+    }
+    if (disableLibraryUnloading)
+    {
+        khrIcd_free_getenv(disableLibraryUnloading);
+    }
 }
 
 // entrypoint to initialize the ICD and add all vendors
@@ -186,6 +205,14 @@ void khrIcdVendorAdd(const char *libraryName)
 #endif
 
         // call clGetPlatformInfo on the returned platform to get the suffix
+
+        KHR_ICD2_DISPATCH(platforms[i])->clGetPlatformInfo(
+            platforms[i],
+            CL_PLATFORM_UNLOADABLE_KHR,
+            sizeof(vendor->unloadable),
+            &vendor->unloadable,
+            NULL);
+
         result = KHR_ICD2_DISPATCH(platforms[i])->clGetPlatformInfo(
             platforms[i],
             CL_PLATFORM_ICD_SUFFIX_KHR,
@@ -230,11 +257,13 @@ void khrIcdVendorAdd(const char *libraryName)
         vendor->suffix = suffix;
 
         // add this vendor to the list of vendors at the tail
-        {
-            KHRicdVendor **prevNextPointer = NULL;
-            for (prevNextPointer = &khrIcdVendors; *prevNextPointer; prevNextPointer = &( (*prevNextPointer)->next) );
-            *prevNextPointer = vendor;
+        if (lastVendor) {
+            lastVendor->next = vendor;
+            vendor->prev = lastVendor;
+        } else {
+            khrIcdVendors = vendor;
         }
+        lastVendor = vendor;
 
         KHR_ICD_TRACE("successfully added vendor %s with suffix %s\n", libraryName, suffix);
 
@@ -259,6 +288,7 @@ void khrIcdLayerAdd(const char *libraryName)
     cl_int result = CL_SUCCESS;
     pfn_clGetLayerInfo p_clGetLayerInfo = NULL;
     pfn_clInitLayer p_clInitLayer = NULL;
+    pfn_clDeinitLayer p_clDeinitLayer = NULL;
     struct KHRLayer *layerIterator = NULL;
     struct KHRLayer *layer = NULL;
     cl_layer_api_version api_version = 0;
@@ -315,12 +345,22 @@ void khrIcdLayerAdd(const char *libraryName)
         goto Done;
     }
 
-    if (CL_LAYER_API_VERSION_100 != api_version)
+    if (CL_LAYER_API_VERSION_100 != api_version && CL_LAYER_API_VERSION_200 != api_version)
     {
         KHR_ICD_TRACE("unsupported api version\n");
         goto Done;
     }
 
+    if (CL_LAYER_API_VERSION_200 == api_version)
+    {
+        p_clDeinitLayer = (pfn_clDeinitLayer)(size_t)khrIcdOsLibraryGetFunctionAddress(library, "clDeinitLayer");
+        if (!p_clDeinitLayer)
+        {
+            KHR_ICD_TRACE("failed to get function address clDeinitLayer\n");
+            goto Done;
+        }
+    }
+
     layer = (struct KHRLayer*)calloc(sizeof(struct KHRLayer), 1);
     if (!layer)
     {
@@ -338,9 +378,10 @@ void khrIcdLayerAdd(const char *libraryName)
             goto Done;
         }
         memcpy(layer->libraryName, libraryName, sz_name);
-        layer->p_clGetLayerInfo = (void *)(size_t)p_clGetLayerInfo;
+        layer->p_clGetLayerInfo = p_clGetLayerInfo;
     }
 #endif
+    layer->p_clDeinitLayer = p_clDeinitLayer;
 
     if (khrFirstLayer) {
         targetDispatch = &(khrFirstLayer->dispatch);
@@ -472,3 +513,45 @@ void khrIcdContextPropertiesGetPlatform(const cl_context_properties *properties,
     }
 }
 
+#if defined(CL_ENABLE_LAYERS)
+static struct KHRLayer deinitLayer = {0};
+#endif
+
+void khrIcdDeinitialize(void) {
+
+    KHR_ICD_TRACE("ICD Loader deinitialization\n");
+
+#if defined(CL_ENABLE_LAYERS)
+    // free layers first in reverse order of their creation (front to back)
+    // they may still need to use vendors while terminating
+    KHR_ICD_TRACE("Finalizing and unloading layers\n");
+    struct KHRLayer *head = khrFirstLayer;
+    deinitLayer.dispatch = khrDeinitDispatch;
+    khrFirstLayer = &deinitLayer;
+
+    while(head) {
+        struct KHRLayer *cur = head;
+#ifdef CL_LAYER_INFO
+        free(cur->libraryName);
+#endif
+        if (cur->p_clDeinitLayer)
+            cur->p_clDeinitLayer();
+        if (cur->p_clDeinitLayer && !khrDisableLibraryUnloading)
+            khrIcdOsLibraryUnload(cur->library);
+        head = cur->next;
+        free(cur);
+    }
+#endif // defined(CL_ENABLE_LAYERS)
+
+    // free vendor in reverse order of their creation (back to front)
+    KHR_ICD_TRACE("Finalizing and unloading vendors\n");
+    while (lastVendor) {
+        KHRicdVendor *cur = lastVendor;
+        free(cur->suffix);
+        if (cur->unloadable && !khrDisableLibraryUnloading)
+            khrIcdOsLibraryUnload(cur->library);
+        lastVendor = cur->prev;
+        free(cur);
+    }
+    khrIcdVendors = NULL;
+}

loader/icd.h

@@ -49,6 +49,9 @@
 #include <CL/cl.h>
 #include <CL/cl_ext.h>
 #include <CL/cl_icd.h>
+#if defined(CL_ENABLE_LAYERS)
+#include <CL/cl_layer.h>
+#endif // defined(CL_ENABLE_LAYERS)
 #include <stdio.h>
 
 /*
@@ -85,6 +88,9 @@ struct KHRicdVendorRec
     // the extension suffix for this platform
     char *suffix;
 
+    // can this vendor library be unloaded?
+    cl_bool unloadable;
+
     // function pointer to the ICD platform IDs extracted from the library
     pfn_clGetExtensionFunctionAddress clGetExtensionFunctionAddress;
 
@@ -98,6 +104,7 @@ struct KHRicdVendorRec
 
     // next vendor in the list vendors
     KHRicdVendor *next;
+    KHRicdVendor *prev;
 };
 
 // the global state
@@ -123,14 +130,17 @@ struct KHRLayer
 #ifdef CL_LAYER_INFO
     // The layer library name
     char *libraryName;
-    // the pointer to the clGetLayerInfo funciton
-    void *p_clGetLayerInfo;
+    // the pointer to the clGetLayerInfo function
+    pfn_clGetLayerInfo p_clGetLayerInfo;
 #endif
+    // the pointer to the clDeinitLayer function
+    pfn_clDeinitLayer p_clDeinitLayer;
 };
 
 // the global layer state
 extern struct KHRLayer * khrFirstLayer;
 extern const struct _cl_icd_dispatch khrMainDispatch;
+extern const struct _cl_icd_dispatch khrDeinitDispatch;
 #endif // defined(CL_ENABLE_LAYERS)
 
 /* 
@@ -147,6 +157,12 @@ void khrIcdInitialize(void);
 // entrypoint to check and initialize trace.
 void khrIcdInitializeTrace(void);
 
+// entrypoint to check and initialize library unloading
+void khrIcdInitializeLibraryUnloading(void);
+
+// entrypoint to release icd resources
+void khrIcdDeinitialize(void);
+
 // go through the list of vendors (in /etc/OpenCL.conf or through 
 // the registry) and call khrIcdVendorAdd for each vendor encountered
 // n.b, this call is OS-specific