dlclose does not close library open file handles

Question

I am dynamically loading a library with dlopen, then closing it with dlclose. I expected all library resources to be freed once dlclose completed, but there are still open file descriptors from the library after the dlclose call. I am wondering how to make sure a library is unloaded in the middle of program execution, such that it cleans up all of its resources.

My code is below:

#include <CL/cl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <dlfcn.h>
#include <string.h>
#include <dirent.h>
#include <fcntl.h>
#include <unistd.h>

#define MAX_PATH_LENGTH         80

int deviceQ()
{
    cl_int ret;
    void * libHandle = dlopen("/usr/lib64/libOpenCL.so", RTLD_LAZY);
    cl_int (* clGetPlatformIDs)(cl_uint, cl_platform_id*, cl_uint*) = dlsym(
            libHandle, "clGetPlatformIDs"
    );
    cl_int (* clGetDeviceIDs)(cl_platform_id, cl_device_type, cl_uint, cl_device_id*, cl_uint*) =
        dlsym(libHandle, "clGetDeviceIDs");
    /********************** PREAMBLE **************************************/
    cl_device_id device_id = NULL;
    cl_platform_id platform_id = NULL;
    cl_uint ret_num_devices;
    cl_uint ret_num_platforms;

    ret = clGetPlatformIDs(1, &platform_id, &ret_num_platforms);
    if (ret != CL_SUCCESS) {
        perror("Failed to get platform IDs");
    } else if (ret_num_platforms != 1) {
        fprintf(stderr, "Number of platforms returned is %d\n", ret_num_platforms);
        exit(1);
    }

    printf("Read platform IDs\n");

    ret = clGetDeviceIDs(platform_id, CL_DEVICE_TYPE_GPU, 1, &device_id,
        &ret_num_devices);
    if (ret != CL_SUCCESS) {
        perror("Failed to get device IDs");
    } else if (ret_num_devices != 1) {
        fprintf(stderr, "Number of returned devices is %d\n", ret_num_devices);
        exit(1);
    }
    printf("Read device IDs\n");
    /********************** PREAMBLE **************************************/

    /***************** RELEASE AND FREE ****************************/
    dlclose(libHandle);
    /***************** RELEASE AND FREE ****************************/

    return 0;
}

size_t closeFileDescriptors(void ** arr) {
    // step 1 - get PID
    pid_t pid = getpid();
    //printf("PID is %d\n", pid);

    char path[MAX_PATH_LENGTH];
    memset(path, '\0', MAX_PATH_LENGTH);
    sprintf(path, "/proc/%d/fd", pid);

    int fd;
    DIR * d = opendir(path);
    struct dirent *dir;
    struct stat s;
    char dirPath[MAX_PATH_LENGTH];
    char realPath[MAX_PATH_LENGTH];

    size_t index = 0;

    if (d) {
        while ((dir = readdir(d)) != NULL) {
            if (strcmp(dir->d_name, ".") != 0 && 
                strcmp(dir->d_name, "..") != 0) {
                fd = atoi(dir->d_name);

                if (fstat(fd, &s) != 0) {
                    perror("fstat failed");
                }
                memset(dirPath, '\0', MAX_PATH_LENGTH);
                strcpy(dirPath, path);
                strcat(dirPath, "/");
                strcat(dirPath, dir->d_name);

                #ifdef S_IFLNK
                if (s.st_mode & S_IFLNK) {
                #else
                if (S_ISLNK(s.st_mode)) {
                #endif
                    memset(realPath, '\0', MAX_PATH_LENGTH);
                    #ifdef readlink
                    readlink(dirPath, realPath, MAX_PATH_LENGTH);
                    printf("%s -> %s\n", dirPath, realPath);
                    #else
                    printf("[readlink not defined] %s\n", dirPath);
                    #endif
                } else {
                    printf("Not link: %s (proceeding anyway)\n", dirPath);
                    //printf("Not link: %s (ignoring)\n", dirPath);
                    //continue;
                }

                if (fd > 2) {
                    //int fdFlags = fcntl(fd, F_GETFD);
                    int fdFlags = fcntl(fd, F_GETFL);
                    if (fdFlags == -1) {
                        perror("fcntl failed");
                    }
                    //off_t offset = lseek(fd, 0, SEEK_CUR);
                    off_t offset = 0;
                    if (offset == -1) {
                        perror("lseek failed");
                    }
                    if (arr != NULL) {
                        /*
                        arr[index] = (fileData *) malloc(sizeof (fileData));
                        arr[index]->flags = fdFlags;
                        arr[index]->offset = offset;
                        arr[index]->fd = fd;
                        strcpy(arr[index]->fdPath, realPath);*/
                    }
                    index++;

                    // ignore stdin, stdout, stderr
                    printf("Closing FD %d (flags %d, offset %zd)\n", 
                            fd, fdFlags, offset);
                    close(fd);
                }
            }
        }
        closedir(d);
    } else {
        fprintf(stderr, "Could not open directory %s\n", path);
    }
    return index;
}

int main () {
    deviceQ();

    printf("=> Closing open file descriptors\n");
    closeFileDescriptors (NULL);

    deviceQ();
    return 0;
}

Answer 1

Your expectation is wrong. When you call dlclose(3), only the "plugin" (actually shared object) is "closed" (actually, may be munmap-ed), but not the resources (in particular the file descriptors, and possibly heap allocated memory) it has used.

In addition, on Linux specifically, dlclose is calling the so-called destructor functions of the plugin (those declared with __attribute__((destructor)), read about function attributes in GCC).

If you are coding a shared library, you might design it so that some resources are released at dlclose time (by having appropriate finalizations run thru destructor functions). In general, it is not easily possible (and it should be a documented convention).

Resources like address space in virtual memory (obtained by mmap(2) etc...) and file descriptors (obtained by open(2), socket(2), pipe(2) etc etc...) are global (and common) to the entire process. So it would be possible (and legitimate, if documented) to acquire some resource (e.g. open some file descriptor) in one shared library and to release it in another one (or in the main program).

Since a resource "belongs" to the entire process, it makes no sense to speak of releasing the resources acquired by a library.

So your closeFileDescriptors is quite probably a huge mistake (and it probably leaks some other resources).

^{(IIRC, OpenCL API has some way to release its resources, e.g. devices, contexts, kernels, etc.... But I forgot the ugly details; see clReleaseContext, clReleaseMemObject and many more, including some implementation specific ones.)}

Reading more about garbage collection would probably widen your mind.

Read also Drepper's paper: How To Write a Shared Library & credentials(7)

If you absolutely need to release OpenCL related resources early, a more sensible way might be to start a different child process dedicated to OpenCL things, and use clever IPC mechanisms (e.g. pipe(7), shm_overview(7), sem_overview(7), etc...) then terminate (properly) that child process once your OpenCL stuff is done. You take advantage of the fact that the kernel is cleaning all the resources used by a defunct process (don't forget to wait... it -e.g. using waitpid(2)- to avoid having zombie processes). If you are not familiar with all that, read Advanced Linux Programming first.