// Copyright (C) 2018-2020 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#define _GNU_SOURCE

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <time.h>
#include <sys/types.h>
#if (defined(_WIN32) || defined(_WIN64))
#include "win_time.h"
#include <windows.h>    // for Sleep()
#include <io.h>
#else
#include <dlfcn.h>      // For dladdr
#include <unistd.h>
#include <dirent.h>
#include <sys/file.h>
#include <errno.h>
#endif

#include "mvnc.h"
#include "mvnc_data.h"

#include "XLink.h"
#include "ncCommPrivate.h"
#include "ncPrivateTypes.h"
#include "XLinkPlatform.h"

#define MVLOG_UNIT_NAME ncAPI
#include "XLinkLog.h"
#include "mvnc_tool.h"
#include "XLinkMacros.h"
#include "XLinkStringUtils.h"
#include "watchdog.h"

#define THERMAL_BUFFER_SIZE 100
#define THERMAL_THROTTLING_BUFFER_SIZE (THERMAL_BUFFER_SIZE + sizeof(int))
#define DEBUG_BUFFER_SIZE     120

#define MAX_TENSORS_TO_LOAD (2)
#define BLOB_STREAM_SIZE 4096
#define TENSOR_STREAM_SIZE 320*1024   * MAX_TENSORS_TO_LOAD
#define OUTPUT_STREAM_SIZE 8 //read only from PC

#define CONFIG_STREAM_SIZE 2000

#define MAX_PATH_LENGTH         255
#define MAX_RELATED_PATH_LENGTH   100

//      Firmware
#define FIRMWARE_DIR_LENGTH         (190)
#define FIRMWARE_NAME_LENGTH        (60)
#define FIRMWARE_PROTOCOL_LENGTH    (15)
#define FIRWMARE_DEVICE_LENGTH      (30)
#define FIRMWARE_FORMAT_LENGTH      (15)

//      Timeouts
#define DEVICE_APPEAR_TIMEOUT_ON_OPEN       (5)
#define DEVICE_APPEAR_TIMEOUT_ON_CLOSE      (10)

#define SLEEP_MS        250
#define MAX_ITERATIONS  20

#define FP16_DATA_SIZE 2

static int initialized = 0;
static int reset_all = 1;

static int g_deviceConnectTimeoutSec = 15;

pthread_mutex_t deviceOpenMutex = PTHREAD_MUTEX_INITIALIZER;

#if (defined(_WIN32) || defined(_WIN64))
static HANDLE global_lock_fd = NULL;
static OVERLAPPED global_lock_overlap = { 0 };
#define GLOBAL_LOCK() LockFileEx(global_lock_fd, LOCKFILE_EXCLUSIVE_LOCK, 0, MAXDWORD, MAXDWORD, &global_lock_overlap)
#define GLOBAL_UNLOCK() UnlockFileEx(global_lock_fd, 0, MAXDWORD, MAXDWORD, &global_lock_overlap)
#else
static int global_lock_fd = -1;
#define GLOBAL_LOCK() flock(global_lock_fd, LOCK_EX)
#define GLOBAL_UNLOCK() flock(global_lock_fd, LOCK_UN)
#endif

#define STRINGIFY(_text) #_text
#define CASE(entry) case entry: return STRINGIFY(entry);


// To suppress warning in the macro below
#pragma GCC diagnostic ignored "-Wformat-extra-args"

/**
 * @brief The macro checks a stream id passed to it
 * @param id Stream id to check
 * @param callReleasingResources if it is needed to release resource in case of error, put your code of releasing
 *        to { you code here }. If no need to release resource pass {} to the parameter
 * @param errorMsg Message to be written in case of error. It is a format string
 */
#ifndef CHECK_STREAM_ID
#define CHECK_STREAM_ID(id, callReleasingResources, errorMsg) {                                                     \
    char errorMsgWithReason[255];                                                                                   \
    if (id == INVALID_STREAM_ID_OUT_OF_MEMORY) {                                                                    \
        snprintf(errorMsgWithReason, 255, "%s %s", errorMsg, "due to not enough memory on device");                 \
        mvLog(MVLOG_ERROR, errorMsgWithReason);                                                                     \
        callReleasingResources;                                                                                     \
        return NC_OUT_OF_MEMORY;                                                                                    \
    } else if (id == INVALID_STREAM_ID) {                                                                           \
         snprintf(errorMsgWithReason, 255, "%s %s", errorMsg, "due to unknown error");                              \
         mvLog(MVLOG_ERROR, errorMsgWithReason);                                                                    \
         callReleasingResources;                                                                                    \
         return NC_ERROR;                                                                                           \
    }                                                                                                               \
    mvLog(MVLOG_DEBUG, "Stream opened");                                                                            \
}
#endif // CHECK_STREAM_ID

static XLinkGlobalHandler_t ghandler;

devicePrivate_t *devices;

/////////////////////////// Converters /////////////////////////////

char* ncProtocolToStr(const ncDeviceProtocol_t deviceProtocol) {
    switch (deviceProtocol) {
        case NC_USB:            return "USB";
        case NC_PCIE:           return "PCIE";
        case NC_ANY_PROTOCOL:   return "ANY_PROTOCOL";
        default:                return "Unknown protocol name";
    }
}

char* ncPlatformToStr(const ncDevicePlatform_t platform) {
    switch(platform) {
        case NC_MYRIAD_2:              return "NC_MYRIAD_2";
        case NC_MYRIAD_X:              return "NC_MYRIAD_X";
        default:                    return "NC_ANY_PLATFORM";
    }
}

int mvnc_memcpy(void* dest, size_t destsz, void const* src, size_t count) {
    size_t i;
    if (!src || count > destsz ||
        count > (dest > src ? ((uintptr_t)dest - (uintptr_t)src)
                            : ((uintptr_t)src - (uintptr_t)dest))) {
        // zero out dest if error detected
        memset(dest, 0, destsz);
        return -1;
    }

    for (i = 0; i < count; ++i) ((uint8_t*)dest)[i] = ((const uint8_t*)src)[i];
    return 0;
}

static
char* mvnc_strdup(const char* s) {
#ifdef _MSC_VER
    return _strdup(s);
#else
    return strdup(s);
#endif
}

static ncStatus_t parseXLinkError(XLinkError_t rc) {
    switch (rc) {
    case X_LINK_SUCCESS:
        return NC_OK;
    case X_LINK_DEVICE_NOT_FOUND:
        return NC_DEVICE_NOT_FOUND;
    case X_LINK_TIMEOUT:
        return NC_TIMEOUT;
    default:
        return NC_ERROR;
    }
}

static char* ncStatusToStr(const ncStatus_t status) {
    switch(status) {
        CASE(NC_OK)
        CASE(NC_BUSY)
        CASE(NC_OUT_OF_MEMORY)
        CASE(NC_DEVICE_NOT_FOUND)
        CASE(NC_INVALID_PARAMETERS)
        CASE(NC_TIMEOUT)
        CASE(NC_MVCMD_NOT_FOUND)
        CASE(NC_NOT_ALLOCATED)
        CASE(NC_UNAUTHORIZED)
        CASE(NC_UNSUPPORTED_GRAPH_FILE)
        CASE(NC_UNSUPPORTED_CONFIGURATION_FILE)
        CASE(NC_UNSUPPORTED_FEATURE)
        CASE(NC_MYRIAD_ERROR)
        CASE(NC_INVALID_DATA_LENGTH)
        CASE(NC_INVALID_HANDLE)
        default: return STRINGIFY(NC_ERROR);
    }
}

static char* ncMvNCIErrorCodeToStr(const ncMvNCIErrorCode_t code) {
    switch(code) {
        CASE(MVNCI_SUCCESS)
        CASE(MVNCI_NULL_PARAM)
        CASE(MVNCI_MASK_NOTCONTINUOUS)
        CASE(MVNCI_UNSUPPORTED_NETWORK_ELEMENT)
        CASE(MVNCI_INVALID_HANDLE)
        CASE(MVNCI_OUT_OF_RESOURCES)
        CASE(MVNCI_NOT_IMPLEMENTED)
        CASE(MVNCI_SHAVES_SLICES_MISMATCH)
        CASE(MVNCI_TIMEOUT)
        CASE(MVNCI_OUT_OF_MEMORY)
        default: return STRINGIFY(MVNCI_INTERNAL_ERROR);
    }
}

static double timeInSeconds()
{
    static double s;
    struct timespec ts;

    clock_gettime(CLOCK_MONOTONIC, &ts);
    if (!s)
        s = ts.tv_sec + ts.tv_nsec * 1e-9;
    return ts.tv_sec + ts.tv_nsec * 1e-9 - s;
}

static void sleepForSeconds(const unsigned int seconds) {
#if (!defined(_WIN32) && !defined(_WIN64))
    sleep(seconds);
#else
    Sleep(seconds * 1000); // Sleep using miliseconds as input
#endif
}

static ncOptionAccess_t getOptionAccess(int option, int base)
{
    return (int) ((option - base) / OPTION_CLASS_SIZE);
}

#if (defined(_WIN32) || defined(_WIN64) )
#define MAX_2(a,b)		((a) > (b) ? (a) : (b))
#define MAX_3(a,b,c)	((a) > (b) ? MAX_2((a), (c)) : MAX_2((b), (c)))
#ifdef MAX
#undef MAX
#define MAX MAX_2
#endif
#else
#define MAX_3(a,b,c)                            \
    ({ __typeof__ (a) _a = (a);                 \
        __typeof__ (b) _b = (b);                \
        __typeof__ (c) _c = (c);                \
        (_a > _b && _a > _c) ? _a : ((_b > _c && _b > _a) ? _b : _c); })
#endif

static void resetAll()
{
#if defined(NO_BOOT)
    mvLog(MVLOG_INFO, "Devices will not be restarted for this configuration (NO_BOOT)");
#else
    // Reset only USB devices
    deviceDesc_t in_deviceDesc = {
        .protocol = X_LINK_ANY_PROTOCOL,
        .platform = X_LINK_ANY_PLATFORM
    };

    unsigned int stalled_count = 0;
    deviceDesc_t stalledDevices[NC_MAX_DEVICES] = { { 0 } };

    unsigned int stalled_count_after_reboot = 0;


    double waittm = timeInSeconds() + DEVICE_APPEAR_TIMEOUT_ON_OPEN;
    do {
        // Find stalled devices
        stalled_count = 0;
        XLinkFindAllSuitableDevices(
                X_LINK_BOOTED, in_deviceDesc, stalledDevices, NC_MAX_DEVICES, &stalled_count);

        if (stalled_count) {
            mvLog(MVLOG_INFO, "%d stalled devices found, Resetting...", stalled_count);
        } else {
            mvLog(MVLOG_DEBUG, "Stalled devices not found");
            return;
        }

        // Try to reboot them
        int i;
        for (i = 0; i < stalled_count; ++i) {
            mvLog(MVLOG_DEBUG, "Found stalled device %s", stalledDevices[i].name);

            XLinkHandler_t* handler = calloc(1, sizeof(XLinkHandler_t));
            if (!handler){
                mvLog(MVLOG_ERROR, "Memory allocation failed");
                return;
            }

            handler->protocol = stalledDevices[i].protocol;
            handler->devicePath = (char*)stalledDevices[i].name;
            XLinkError_t rc = XLinkConnect(handler);
            if (rc) {
                mvLog(MVLOG_ERROR," Failed to connect to stalled device, rc: %s", XLinkErrorToStr(rc));
            } else {

            }
            free(handler);
        }

        // This command will reset all previously connected devices
        XLinkError_t rc = XLinkResetAll();
        if (rc) {
            mvLog(MVLOG_WARN,"Failed to reset all device, rc: %s", XLinkErrorToStr(rc));
        }

        // Check that all devices are rebooted
        stalled_count_after_reboot = 0;
        deviceDesc_t stalledDevicesAfterReboot[NC_MAX_DEVICES] = { { 0 } };
        XLinkFindAllSuitableDevices(
                X_LINK_BOOTED, in_deviceDesc,
                stalledDevicesAfterReboot, NC_MAX_DEVICES, &stalled_count_after_reboot);

        mvLog(MVLOG_INFO,"...");
        usleep(SLEEP_MS*1000);

    } while (stalled_count_after_reboot > 0 && timeInSeconds() < waittm);
#endif
}

static ncStatus_t initializeXLink();

ncStatus_t ncDeviceResetAll() {
#if defined(NO_BOOT)
    mvLog(MVLOG_INFO, "Devices will not be restarted for this configuration (NO_BOOT)");
#else
    if (!initialized) {
        ncStatus_t sc;
        if ((sc = initializeXLink()) != 0) {
            return sc;
        }
    }
    resetAll();
#endif
    return NC_OK;
}

static ncStatus_t initializeXLink()
{
    XLinkSetCommonTimeOutMsec(60 * 1000);
    // We sanitize the situation by trying to reset the devices that have been left open
    initialized = 1;
    devices = NULL;

    int sc = XLinkInitialize(&ghandler);
    if (sc != X_LINK_SUCCESS) {
        mvLog(MVLOG_ERROR," Initialization failed, rc = %s\n", XLinkErrorToStr(sc));
        return NC_ERROR;
    }

#if !(defined(NO_BOOT))
    if (reset_all) {
        resetAll();
    }
#endif  // NO_BOOT
    return NC_OK;
}

/**
 * @brief Check is path exists (directory or file)
 */
static int isPathExists(const char* filePath) {
#if (defined(_WIN32) || defined(_WIN64))
    return ( _access( filePath, 0 ) != -1 ) ? 1 : 0;
#else
    return (  access( filePath, 0 ) != -1 ) ? 1 : 0;
#endif
}

static char getPathSeparator() {
#ifdef _WIN32
    return '\\';
#else
    return '/';
#endif
}

/**
 * @brief Add / or \\ at the end of the path, if doesn't have it
 */

static void addEndPathSeparator(char* buffer, const int buffer_length) {
    const int filePathLen = strnlen(buffer, buffer_length);
    if ((filePathLen > 1) && (filePathLen < buffer_length - 1) &&
            buffer[filePathLen - 1] != getPathSeparator()) {
        buffer[filePathLen] = getPathSeparator();
        buffer[filePathLen + 1] = 0;
    }
}

static char* getProtocolName(XLinkProtocol_t protocol) {
    if (protocol == X_LINK_PCIE) {
        return "pcie";
    } else if (protocol == X_LINK_USB_VSC) {
        return "usb";
    }
    return "";
}

static ncStatus_t getDeviceFwProtocolPrefix(const deviceDesc_t deviceDesc,
                                            char *fw_protocol_prefix,
                                            const int fw_protocol_prefix_length) {
    if (deviceDesc.protocol != X_LINK_USB_VSC && deviceDesc.protocol != X_LINK_PCIE) {
        return NC_INVALID_PARAMETERS;
    }

    int rc = mv_strcpy(fw_protocol_prefix, fw_protocol_prefix_length,
                                getProtocolName(deviceDesc.protocol));
    if (rc != 0) {
        return NC_ERROR;
    }
    return NC_OK;
}

static char* getDevicePlatform(deviceDesc_t deviceDesc, int useUniversalFirmware) {
    if (deviceDesc.platform == X_LINK_MYRIAD_X) {
        if (useUniversalFirmware && deviceDesc.protocol != X_LINK_PCIE) {
            return "ma2x8x";
        } else {
            return "ma248x";
        }
    } else if (deviceDesc.platform == X_LINK_MYRIAD_2) {
        return "ma2450";
    }
    return "";
}

static ncStatus_t getDeviceFwNameBody(const deviceDesc_t deviceDesc,
                                      char *fw_device_name,
                                      const int fw_device_name_length,
                                      const int useUniversalFirmware) {
    if (deviceDesc.platform != X_LINK_MYRIAD_2 && deviceDesc.platform != X_LINK_MYRIAD_X) {
        return NC_INVALID_PARAMETERS;
    }

    int rc = mv_strcpy(fw_device_name, fw_device_name_length,
                                getDevicePlatform(deviceDesc, useUniversalFirmware));
    if (rc != 0) {
        return NC_ERROR;
    }
    return NC_OK;
}

static ncStatus_t getDeviceFwFormat(const deviceDesc_t deviceDesc,
                                    char *fw_format, const int fw_formate_length) {
    // On Windows unified bootloader .elf file required instead of mvcmd
    if (deviceDesc.protocol == X_LINK_PCIE) {
#if defined(_WIN32)
        mv_strcpy(fw_format, fw_formate_length, ".elf");
#else
        mv_strcpy(fw_format, fw_formate_length, ".mvcmd");
#endif //defined(_WIN32)
    } else {
        mv_strcpy(fw_format, fw_formate_length, ".mvcmd");
    }
    return NC_OK;
}

static ncStatus_t getLibDirectory(char *firmware_directory, const int firwmare_directory_length) {
    // If firmware_directory contain path, use it.
    // It's case when firmware_directory was set by ncDeviceOpen custom path argument
    if (!strnlen(firmware_directory, firwmare_directory_length)) {
        int rc = 0;
        char path_to_lib_file[MAX_PATH_LENGTH] = {0};
        // Get dll full path
#if (defined(_WIN32) || defined(_WIN64))
        HMODULE hm = NULL;
        if (!GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
                                  GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
                              (LPCSTR) "ncDeviceOpen", &hm)) {
            int ret = GetLastError();
            fprintf(stderr, "GetModuleHandle returned %d", ret);
        }
        GetModuleFileNameA(hm, path_to_lib_file, MAX_PATH_LENGTH - 1);
#else
        Dl_info info;
        dladdr(ncDeviceOpen, &info);
        rc = mv_strncpy(path_to_lib_file, MAX_PATH_LENGTH, info.dli_fname, MAX_PATH_LENGTH - 1);
        if (rc != 0) {
            return NC_ERROR;
        }
#endif
        // Path can contains library name. Use path before last '/'
        char* pointerToSeparator = NULL;
        size_t lib_dir_path_len = 0;

        pointerToSeparator = strrchr(path_to_lib_file, getPathSeparator());
        if(pointerToSeparator) {
            *pointerToSeparator = 0;
            lib_dir_path_len = pointerToSeparator - path_to_lib_file + 1;
        }

        rc = mv_strncpy(firmware_directory, firwmare_directory_length, path_to_lib_file, lib_dir_path_len);
        if (rc != 0) {
            return NC_ERROR;
        }
    }
    addEndPathSeparator(firmware_directory, firwmare_directory_length);
    return NC_OK;
}

static int isDeviceDescriptionCorrect(deviceDesc_t deviceDesc) {
    if (strnlen(deviceDesc.name, XLINK_MAX_NAME_SIZE) == 0) {
        mvLog(MVLOG_INFO, "Device name is empty");
        return 0;
    }

    //
    if (strstr(deviceDesc.name, "ma") && deviceDesc.protocol != X_LINK_USB_VSC) {
        mvLog(MVLOG_INFO, "Mismatch device name and protocol. Device name: %s. Protocol: %s",
              deviceDesc.name, ncProtocolToStr(convertProtocolToNC(deviceDesc.protocol)));
        return 0;
    }

    // PCIe devices should be named mxlk/mxlink
    if (strstr(deviceDesc.name, "mxl") && deviceDesc.protocol != X_LINK_PCIE) {
        mvLog(MVLOG_INFO, "Mismatch device name and protocol. Device name: %s. Protocol: %s",
                deviceDesc.name, ncProtocolToStr(convertProtocolToNC(deviceDesc.protocol)));
        return 0;
    }

    // Myriad 2 PCIe is not exists
    if ((strstr(deviceDesc.name, "2150") || deviceDesc.platform == X_LINK_MYRIAD_2)
                            && deviceDesc.protocol == X_LINK_PCIE) {
        mvLog(MVLOG_INFO, "Incorrect platform for PCIe device");
        return 0;
    }

    if (deviceDesc.protocol != X_LINK_ANY_PROTOCOL &&
        deviceDesc.protocol != X_LINK_USB_VSC &&
        deviceDesc.protocol != X_LINK_PCIE) {
        mvLog(MVLOG_INFO, "Protocol %s not supported",
                ncProtocolToStr(convertProtocolToNC(deviceDesc.protocol)));
        return 0;
    }
    return 1;
}

/**
 * Search the mvnc executable in the same directory of this library
 * in the future there will ideally be one FW file for all, for now they are separate
 */
ncStatus_t getFirmwarePath(char *firmware_file_path, const int firmware_file_length,
                           const deviceDesc_t deviceDesc) {

    if (!firmware_file_path || !isDeviceDescriptionCorrect(deviceDesc) ||
        deviceDesc.protocol == X_LINK_ANY_PROTOCOL) {
        return NC_INVALID_PARAMETERS;
    }

    ncStatus_t ncStatus;
    int rc;

    int useUniversalFirmware = 1;       // Try to use universal first

    char full_path_to_firmware[MAX_PATH_LENGTH]           = {0};

    char firmware_dir[FIRMWARE_DIR_LENGTH]                = {0};

    char firmware_full_name[FIRMWARE_NAME_LENGTH]         = {0};
    char fw_protocol_prefix[FIRMWARE_PROTOCOL_LENGTH]     = {0};
    char fw_device_name[FIRWMARE_DEVICE_LENGTH]           = {0};
    char fw_format[FIRMWARE_FORMAT_LENGTH]                = {0};

    // User can provide custom directory with firmware
    rc = mv_strncpy(firmware_dir, FIRMWARE_DIR_LENGTH, firmware_file_path,
                    strnlen(firmware_file_path, firmware_file_length));
    firmware_file_path[0] = 0;  // Clean input path

    if (rc != 0) {
        return NC_ERROR;
    }

    /// Construct file name
    ncStatus = getDeviceFwProtocolPrefix(deviceDesc, fw_protocol_prefix, FIRMWARE_PROTOCOL_LENGTH);
    if (ncStatus != NC_OK) {
        return ncStatus;
    }

    ncStatus = getDeviceFwNameBody(deviceDesc, fw_device_name, FIRWMARE_DEVICE_LENGTH,
                                   useUniversalFirmware);
    if (ncStatus != NC_OK) {
        return ncStatus;
    }

    ncStatus = getDeviceFwFormat(deviceDesc, fw_format, FIRMWARE_FORMAT_LENGTH);
    if (ncStatus != NC_OK) {
        return ncStatus;
    }

    rc = snprintf(firmware_full_name, FIRMWARE_NAME_LENGTH,
             "%s-%s%s", fw_protocol_prefix, fw_device_name, fw_format);
    if (rc < 0) {
        return NC_ERROR;
    }
    mvLog(MVLOG_DEBUG, "Firmware name %s", firmware_full_name);

    ///     Get file location
    ncStatus = getLibDirectory(firmware_dir, FIRMWARE_DIR_LENGTH);
    if (ncStatus != NC_OK) {
        return ncStatus;
    }
    mvLog(MVLOG_DEBUG, "Firmware dir %s", firmware_full_name);

    rc = snprintf(full_path_to_firmware, MAX_PATH_LENGTH, "%s%s", firmware_dir, firmware_full_name);
    if (rc < 0) {
        return NC_ERROR;
    }

    // If there is no universal firmware available, use a special one
    if (deviceDesc.protocol == X_LINK_USB_VSC && deviceDesc.platform == X_LINK_MYRIAD_X
                                                && !isPathExists(full_path_to_firmware)) {
        mvLog(MVLOG_INFO, "Cannot find universal firmware for ma2x8x. Try to find special one.");

        useUniversalFirmware = 0;
        ncStatus = getDeviceFwNameBody(deviceDesc, fw_device_name, FIRWMARE_DEVICE_LENGTH,
                                       useUniversalFirmware);
        if (ncStatus != NC_OK) {
            return ncStatus;
        }
        rc = snprintf(full_path_to_firmware, MAX_PATH_LENGTH,
                 "%s%s-%s%s", firmware_dir, fw_protocol_prefix, fw_device_name, fw_format);
        if (rc < 0) {
            return NC_ERROR;
        }
    }

    if (!isPathExists(full_path_to_firmware)) {
        mvLog(MVLOG_ERROR, "Firmware not found in: %s", full_path_to_firmware);
        return NC_ERROR;
    }
    rc = mv_strcpy(firmware_file_path, MAX_PATH_LENGTH, full_path_to_firmware);
    if (rc != 0) {
        return NC_ERROR;
    }

    mvLog(MVLOG_DEBUG, "File path %s", firmware_file_path);
    return 0;
}

static ncStatus_t getDevAttributes(struct _devicePrivate_t *d);
static void printfOverXLinkOpen(struct _devicePrivate_t *d);
static void printfOverXLinkClose(struct _devicePrivate_t *d);
static ncStatus_t destroyDeviceHandle(struct ncDeviceHandle_t **deviceHandlePtr);

ncStatus_t ncSetDeviceConnectTimeout(int deviceConnectTimeoutSec) {
    if(deviceConnectTimeoutSec < 0) {
        return NC_INVALID_PARAMETERS;
    }

    g_deviceConnectTimeoutSec = deviceConnectTimeoutSec;
    return NC_OK;
}

ncStatus_t ncDeviceOpen(struct ncDeviceHandle_t **deviceHandlePtr,
    struct ncDeviceDescr_t in_ncDeviceDesc, int watchdogInterval, const char* customFirmwareDirectory) {

    //----------------------------------------------------------
    //      Check input

    deviceDesc_t deviceDescToBoot = {0};
    deviceDesc_t in_deviceDesc = {0};
    copyNcDeviceDescrToXLink(&in_ncDeviceDesc, &in_deviceDesc);

    CHECK_HANDLE_CORRECT_RC(deviceHandlePtr, NC_INVALID_PARAMETERS);
    if (watchdogInterval < 0) {
        mvLog(MVLOG_ERROR, "Invalid watchdogInterval");
        return NC_INVALID_PARAMETERS;
    }

#ifdef NO_BOOT
    XLinkDeviceState_t state = X_LINK_BOOTED;
    if (watchdogInterval > 0) {
        mvLog(MVLOG_INFO, "Watchdog for already booted device would be disabled");
        watchdogInterval = 0;
    }

    // If trying open already booted device, we should not reset_all device on
    mvLog(MVLOG_INFO, "Connect to already booted device");
    reset_all = 0;
#else
    XLinkDeviceState_t state = X_LINK_UNBOOTED;
#endif

    if(!XLinkPlatformIsDescriptionValid(&in_deviceDesc, state)) {
        mvLog(MVLOG_ERROR, "Invalid in_ncDeviceDesc");
        return NC_INVALID_PARAMETERS;
    }

    if (*deviceHandlePtr && (*deviceHandlePtr)->private_data->state == NC_DEVICE_OPENED) {
        mvLog(MVLOG_WARN, "Device was already opened");
        return NC_OK;
    }

    //--------------------------------------------------------
    //      Initialize global mutex and mutex for deviceOpen

    if (!initialized) {
#if (defined(_WIN32) || defined(_WIN64))
        char* tempPath = getenv("TEMP");
        if (tempPath) {
            size_t pathSize = strlen(tempPath) + 15;
            char *path = malloc(pathSize);
            if (!path) {
                return NC_OUT_OF_MEMORY;
            }
            mv_strcpy(path, pathSize, tempPath);
            strcat_s(path, pathSize, "\\mvnc.mutex");
            global_lock_fd = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
            free(path);
        }

        if (!global_lock_fd) {
            mvLog(MVLOG_ERROR, "global mutex initialization failed");
            exit(1);
        }
#else
        global_lock_fd = open(
#if defined(ANDROID)
            "/data/local/tmp/mvnc.mutex",
#else
            "/tmp/mvnc.mutex",
#endif
            O_CREAT, 0660);

        if (global_lock_fd == -1) {
            mvLog(MVLOG_ERROR, "global mutex initialization failed");
            exit(1);
        }
#endif
    }

    GLOBAL_LOCK();
    int error = pthread_mutex_lock(&deviceOpenMutex);
    if (error) {
        GLOBAL_UNLOCK();
        mvLog(MVLOG_ERROR, "pthread_mutex_lock(&deviceOpenMutex) failed with error: %d", error);
        return NC_ERROR;
    }

    if (!initialized) {
        ncStatus_t sc;
        if ((sc = initializeXLink()) != 0) {
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
            GLOBAL_UNLOCK();
            return sc;
        }
    }

    //--------------------------------------------------------
    //      Search for device

    XLinkError_t rc = X_LINK_ERROR;
    double waittm = timeInSeconds() + DEVICE_APPEAR_TIMEOUT_ON_OPEN;
    while ((rc != X_LINK_SUCCESS) && (timeInSeconds() < waittm)) {
        rc = XLinkFindFirstSuitableDevice(state, in_deviceDesc, &deviceDescToBoot);
    }

    if (rc != X_LINK_SUCCESS) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
        GLOBAL_UNLOCK();
        return parseXLinkError(NC_ERROR);
    }

    //--------------------------------------------------------
    //      Allocate device handler

    struct ncDeviceHandle_t *dH = calloc(1, sizeof(*dH));
    struct _devicePrivate_t *d = calloc(1, sizeof(*d));

    if (dH && d) {
        dH->private_data = d;
        d->protocol = deviceDescToBoot.protocol;
        d->dev_addr = mvnc_strdup(deviceDescToBoot.name);
        d->device_mon_stream_id = INVALID_LINK_ID;
        d->graph_monitor_stream_id = INVALID_LINK_ID;
        d->wd_interval = watchdogInterval;
        *deviceHandlePtr = dH;
    } else {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
        GLOBAL_UNLOCK();
        mvLog(MVLOG_ERROR, "Memory allocation failed");
        free(d);
        free(dH);
        return NC_OUT_OF_MEMORY;
    }

    if (d->dev_addr == NULL) {
        destroyDeviceHandle(deviceHandlePtr);
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
        GLOBAL_UNLOCK();
        return NC_OUT_OF_MEMORY;
    }

    //--------------------------------------------------------
    //      Boot device

    XLinkHandler_t* handler = calloc(1, sizeof(XLinkHandler_t));
    if (!handler) {
        mvLog(MVLOG_ERROR, "Memory allocation failed");
        destroyDeviceHandle(deviceHandlePtr);
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
        GLOBAL_UNLOCK();
        return NC_OUT_OF_MEMORY;
    }

    handler->protocol = d->protocol;
    handler->devicePath = (char*)d->dev_addr;


#if (defined(NO_BOOT))
    d->protocol_booted = d->protocol;
    d->dev_addr_booted = mvnc_strdup(d->dev_addr);
    handler->protocol = d->protocol_booted;
    handler->devicePath = d->dev_addr_booted;
    rc = XLinkConnect(handler);
#else
    if (handler->protocol == X_LINK_PCIE) {                             // PCIe
        ncStatus_t sc;
        char mv_cmd_file_path[MAX_PATH_LENGTH] = { 0 };

        if (customFirmwareDirectory && strnlen(customFirmwareDirectory, MAX_PATH_LENGTH) > 1) {
            mv_strncpy(mv_cmd_file_path, MAX_PATH_LENGTH, customFirmwareDirectory, MAX_PATH_LENGTH - 1);
            addEndPathSeparator(mv_cmd_file_path, MAX_PATH_LENGTH);
            mv_cmd_file_path[MAX_PATH_LENGTH - 1] = '\0';
        }

        if ((sc = getFirmwarePath(mv_cmd_file_path, MAX_PATH_LENGTH, deviceDescToBoot)) != 0) {
            mvLog(MVLOG_ERROR, "Can't get firmware, error: %s", ncStatusToStr(sc));
            free(handler);
            destroyDeviceHandle(deviceHandlePtr);
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
            GLOBAL_UNLOCK();
            return NC_MVCMD_NOT_FOUND;
        }

        rc = XLinkBoot(&deviceDescToBoot, mv_cmd_file_path);
        if (rc) {
            mvLog(MVLOG_WARN, "%s() XLinkBootRemote returned error %s for %s",
                  __func__, XLinkErrorToStr(rc), d->dev_addr);
            free(handler);
            destroyDeviceHandle(deviceHandlePtr);
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
            GLOBAL_UNLOCK();
            return NC_ERROR;
        } else {
            mvLog(MVLOG_INFO, "%s() XLinkBootRemote returned success %s for %s",
                  __func__, XLinkErrorToStr(rc), d->dev_addr);
        }
        // Search and connect for booted device
        deviceDesc_t tempDeviceDesc = { 0 };

        d->protocol_booted = d->protocol;
        d->dev_addr_booted = mvnc_strdup(d->dev_addr);
        handler->protocol = d->protocol_booted;
        handler->devicePath = d->dev_addr_booted;

        int isDeviceAppeared = 0;
        int isDeviceConnected = 0;
        waittm = timeInSeconds() + g_deviceConnectTimeoutSec;
        do {
            if(!isDeviceAppeared) {
                rc = XLinkFindFirstSuitableDevice(X_LINK_BOOTED,
                    deviceDescToBoot, &tempDeviceDesc);

                if(rc != X_LINK_SUCCESS) {
                    continue;
                } else {
                    isDeviceAppeared = 1;
                }
            }

            rc = XLinkConnect(handler);
            if(rc == X_LINK_SUCCESS) {
                isDeviceConnected = 1;
            }

            if(isDeviceAppeared && isDeviceConnected) {
                break;
            }
        } while(timeInSeconds() < waittm);

        if (!isDeviceAppeared) {
            mvLog(MVLOG_ERROR, "Failed to find booted device after boot");
        }
    } else {                                        // USB
        // Find firmware and boot device with it
        char mv_cmd_file_path[MAX_PATH_LENGTH] = { 0 };

        // If have firmware directory path as function input, use it
        if (customFirmwareDirectory && strnlen(customFirmwareDirectory, MAX_PATH_LENGTH) > 1) {
            mv_strncpy(mv_cmd_file_path, MAX_PATH_LENGTH, customFirmwareDirectory, MAX_PATH_LENGTH - 1);
            addEndPathSeparator(mv_cmd_file_path, MAX_PATH_LENGTH);
        }

        ncStatus_t sc;

        if ((sc = getFirmwarePath(mv_cmd_file_path, MAX_PATH_LENGTH, deviceDescToBoot)) != 0) {
            mvLog(MVLOG_ERROR, "Can't get firmware, error: %s", ncStatusToStr(sc));
            free(handler);
            destroyDeviceHandle(deviceHandlePtr);
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
            GLOBAL_UNLOCK();
            return NC_MVCMD_NOT_FOUND;
        }

        mvLog(MVLOG_INFO, "%s() XLinkBootRemote is running for %s...\n", __func__, d->dev_addr);

        // Remember all currently available devices
        deviceDesc_t beforeBootDevices[NC_MAX_DEVICES] = {{0}};
        unsigned int numberOfDevicesBeforeBoot = 0;
        deviceDesc_t deviceDesc = {
            .platform = X_LINK_ANY_PLATFORM,
            .protocol = X_LINK_USB_VSC
        };

        XLinkFindAllSuitableDevices(X_LINK_ANY_STATE, deviceDesc, beforeBootDevices,
                                    NC_MAX_DEVICES, &numberOfDevicesBeforeBoot);

        // Boot device
        rc = XLinkBoot(&deviceDescToBoot, mv_cmd_file_path);
        if (rc) {
            mvLog(MVLOG_WARN, "%s() XLinkBootRemote returned error %s for %s",
                  __func__, XLinkErrorToStr(rc), d->dev_addr);
        } else {
            mvLog(MVLOG_INFO, "%s() XLinkBootRemote returned success %s for %s",
                  __func__, XLinkErrorToStr(rc), d->dev_addr);
        }

        // After boot name should change. Find
        deviceDesc_t foundBootedDevice = {0};
        int found_new_booted_device = 0;
        int device_disappear        = 0;
        int isDeviceConnected       = 0;

        deviceDesc_t afterBootDevices[NC_MAX_DEVICES] = { { 0 } };
        unsigned int numberOfDevicesAfterBoot = 0;

        waittm = timeInSeconds() + g_deviceConnectTimeoutSec;
        do {
            if(!found_new_booted_device) {
                XLinkFindAllSuitableDevices(X_LINK_ANY_STATE, deviceDesc, afterBootDevices,
                                            NC_MAX_DEVICES, &numberOfDevicesAfterBoot);
                if (numberOfDevicesAfterBoot != numberOfDevicesBeforeBoot) {
                    continue;
                }
                deviceDesc_t tempDevicDescr = { 0 };

                // Device should disappear from unbooted list
                if (X_LINK_DEVICE_NOT_FOUND != XLinkFindFirstSuitableDevice(
                                                  X_LINK_ANY_STATE, deviceDescToBoot, &tempDevicDescr)) {
                    continue;
                } else {
                    device_disappear = 1;
                }
                int i, j;
                for (i = 0; i < numberOfDevicesAfterBoot; ++i) {
                    int found_in_before_boot_list = 0;
                    for (j = 0; j < numberOfDevicesBeforeBoot; ++j) {
                        if(strcmp(afterBootDevices[i].name, beforeBootDevices[j].name) == 0) {
                            found_in_before_boot_list = 1;
                        }
                    }
                    if (!found_in_before_boot_list) {
                        mv_strcpy(foundBootedDevice.name, XLINK_MAX_NAME_SIZE,
                                  afterBootDevices[i].name);
                        foundBootedDevice.platform = afterBootDevices[i].platform;
                        foundBootedDevice.protocol = afterBootDevices[i].protocol;
                        found_new_booted_device = 1;
                    }
                }
            }

            if(!found_new_booted_device) {
                continue;
            }

            d->protocol_booted = d->protocol;
            d->dev_addr_booted = mvnc_strdup(foundBootedDevice.name);

            handler->protocol = foundBootedDevice.protocol;
            handler->devicePath = (char *) foundBootedDevice.name;

            rc = XLinkConnect(handler);
            if(rc == X_LINK_SUCCESS) {
                isDeviceConnected = 1;
            }

            if(found_new_booted_device && isDeviceConnected) {
                break;
            }

        } while (timeInSeconds() < waittm);

        if (!found_new_booted_device) {
            mvLog(MVLOG_ERROR, "Failed to find booted device after boot");
            if (!device_disappear) {
                mvLog(MVLOG_WARN, "Device (%s) doesn't disappear after firmware loading",
                      deviceDescToBoot.name);
            }
            free(handler);
            destroyDeviceHandle(deviceHandlePtr);
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
            GLOBAL_UNLOCK();
            return NC_ERROR;
        }
    }
#endif

    if (rc != X_LINK_SUCCESS) {
        mvLog(MVLOG_ERROR, "Failed connection to device (%s) with error %d", d->dev_addr, rc);
        free(handler);
        destroyDeviceHandle(deviceHandlePtr);
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
        GLOBAL_UNLOCK();
        return parseXLinkError(rc);
    }

    // After this line calling free(handler) and destroyDeviceHandle after each other is double-free corruption
    d->xlink = handler;
    d->next = devices;

    // Check device handle
    if (d->dev_addr == NULL || d->dev_addr_booted == NULL || d->xlink == NULL) {
        mvLog(MVLOG_ERROR, "device is invalid");
        destroyDeviceHandle(deviceHandlePtr);
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
        GLOBAL_UNLOCK();
        return NC_INVALID_HANDLE;
    }

    devices = d;

    mvLog(MVLOG_INFO, "XLinkConnect done - link Id %d\n", handler->linkId);

    if ((error = pthread_mutex_init(&d->dev_data_m, NULL)) != 0) {
        mvLog(MVLOG_ERROR, "pthread_mutex_init (dev_data_m) failed with error: %d", error);
        destroyDeviceHandle(deviceHandlePtr);
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
        GLOBAL_UNLOCK();
        return NC_ERROR;
    }
    // If current mutex initialization failed, destroy previous
    if ((error = pthread_mutex_init(&d->dev_stream_m, NULL)) != 0) {
        mvLog(MVLOG_ERROR, "pthread_mutex_init (dev_stream_m) failed with error: %d", error);
        CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->dev_data_m));
        destroyDeviceHandle(deviceHandlePtr);
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
        GLOBAL_UNLOCK();
        return NC_ERROR;
    }
    if ((error = pthread_mutex_init(&d->graph_stream_m, NULL)) != 0) {
        mvLog(MVLOG_ERROR, "pthread_mutex_init (graph_stream_m) failed with error: %d", error);
        CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->dev_data_m));
        CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->dev_stream_m));
        destroyDeviceHandle(deviceHandlePtr);
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
        GLOBAL_UNLOCK();
        return NC_ERROR;
    }

    if (handler->protocol != X_LINK_PCIE) {
        mvLog(MVLOG_INFO, "Booted %s (%s) -> %s\n",
              d->dev_addr, d->dev_addr_booted,
              d->dev_file ? d->dev_file : "VSC");
    } else {
        mvLog(MVLOG_INFO, "Booted %s -> %s\n",
              d->dev_addr, d->dev_file ? d->dev_file : "PCIe");
    }

    sleepForSeconds(1);

    CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&deviceOpenMutex));
    GLOBAL_UNLOCK();

    streamId_t deviceMonitorStreamId = XLinkOpenStream(d->xlink->linkId, "deviceMonitor", CONFIG_STREAM_SIZE);
    CHECK_STREAM_ID(
        deviceMonitorStreamId,
        {
            CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->graph_stream_m));
            CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->dev_stream_m));
            CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->dev_data_m));
            destroyDeviceHandle(deviceHandlePtr);
        },
        "can't open deviceMonitor stream");

    d->device_mon_stream_id = deviceMonitorStreamId;

#if !(defined(NO_BOOT))
    watchdog_init_context(&d->watchdog_ctx);
    watchdog_register_device(&d->watchdog_ctx, d);
#endif

    getDevAttributes(d);

#if (!defined(_WIN32) && !defined(_WIN64) && !defined(ANDROID))
    printfOverXLinkOpen(d);
#endif

    streamId_t graphMonitorStreamId = XLinkOpenStream(d->xlink->linkId, "graphMonitor", BLOB_STREAM_SIZE);

#if (!defined(_WIN32) && !defined(_WIN64) && !defined(ANDROID))
    CHECK_STREAM_ID(graphMonitorStreamId, {
        printfOverXLinkClose(d);
        // TODO NO_BOOT case
        watchdog_unregister_device(&d->watchdog_ctx);
        CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->dev_data_m));
        CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->dev_stream_m));
        CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->graph_stream_m));
        XLinkError_t closed = XLinkCloseStream(deviceMonitorStreamId);
        if (closed != X_LINK_SUCCESS) {
            mvLog(MVLOG_ERROR, "Failed to close deviceMonitor stream");
        }

        destroyDeviceHandle(deviceHandlePtr);
    }, "can't open graphMonitor stream");
#else
    CHECK_STREAM_ID(graphMonitorStreamId, {
        // TODO NO_BOOT case
        watchdog_unregister_device(&d->watchdog_ctx);
        CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->dev_data_m));
        CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->dev_stream_m));
        CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->graph_stream_m));
        XLinkError_t closed = XLinkCloseStream(deviceMonitorStreamId);
        if (closed != X_LINK_SUCCESS) {
            mvLog(MVLOG_ERROR, "Failed to close deviceMonitor stream");
        }

        destroyDeviceHandle(deviceHandlePtr);
    }, "can't open graphMonitor stream");
#endif

    d->graph_monitor_stream_id = graphMonitorStreamId;
    d->state = NC_DEVICE_OPENED;

    return NC_OK;
}

ncStatus_t ncAvailableDevices(struct ncDeviceDescr_t *deviceDescrPtr,
                              int maxDevices, int* out_countDevices) {
    CHECK_HANDLE_CORRECT(deviceDescrPtr);
    CHECK_HANDLE_CORRECT(out_countDevices);

    XLinkPlatformInit();
    memset(deviceDescrPtr, 0, maxDevices * sizeof(struct ncDeviceDescr_t));

    deviceDesc_t in_deviceDsc = {
        .platform = X_LINK_ANY_PLATFORM,
        .protocol = X_LINK_ANY_PROTOCOL
    };

    deviceDesc_t deviceDescArray[NC_MAX_DEVICES] = { { 0 } };
    unsigned int amountOfFoundDevices = 0;
    XLinkFindAllSuitableDevices(
            X_LINK_UNBOOTED, in_deviceDsc, deviceDescArray, NC_MAX_DEVICES, &amountOfFoundDevices);
    int i;
    for (i = 0; i < amountOfFoundDevices; ++i) {
        copyXLinkDeviceDescrToNc(&deviceDescArray[i], &deviceDescrPtr[i]);
    }

    *out_countDevices = amountOfFoundDevices;
    return NC_OK;
}

ncStatus_t ncDeviceLoadFirmware(const ncDevicePlatform_t devicePlatform, const char* customFirmwareDir) {
    mvLog(MVLOG_WARN, "Boot (%s) without connecting to it", ncPlatformToStr(devicePlatform));
    XLinkError_t rc;
    ncStatus_t sc;

    // Find device with specific platform
    deviceDesc_t deviceDesc = {0};
    deviceDesc_t in_deviceDesc = {
        .platform = convertPlatformToXlink(devicePlatform),
        .protocol = X_LINK_USB_VSC
    };

    rc = XLinkFindFirstSuitableDevice(X_LINK_UNBOOTED, in_deviceDesc, &deviceDesc);
    if (rc) {
        mvLog(MVLOG_WARN, "Failed to find (%s) platform device", ncPlatformToStr(devicePlatform));
        return NC_DEVICE_NOT_FOUND;
    }

    if(deviceDesc.protocol == X_LINK_PCIE)
    {
        mvLog(MVLOG_WARN, "Firmware for PCIe can't be loaded with this application");
        return NC_ERROR;
    }

    // Find firmware
    char mv_cmd_file_path[MAX_PATH_LENGTH] = "\0";
    if (customFirmwareDir && strnlen(customFirmwareDir, MAX_PATH_LENGTH) > 1) {
        mv_strncpy(mv_cmd_file_path, MAX_PATH_LENGTH, customFirmwareDir, MAX_PATH_LENGTH - 1);
        addEndPathSeparator(mv_cmd_file_path, MAX_PATH_LENGTH);
        if (!isPathExists(customFirmwareDir)) {
            return NC_MVCMD_NOT_FOUND;
        }
    }

    if ((sc = getFirmwarePath(mv_cmd_file_path, MAX_PATH_LENGTH, deviceDesc)) != 0) {
        mvLog(MVLOG_ERROR, "Can't get firmware, error: %s", ncStatusToStr(sc));
        return NC_MVCMD_NOT_FOUND;
    }

    mvLog(MVLOG_INFO, "Trying to boot %s device", deviceDesc.name);
    rc = XLinkBoot(&deviceDesc, mv_cmd_file_path);
    if (rc) {
        mvLog(MVLOG_WARN, "%s() XLinkBootRemote returned error %s\n", __func__, XLinkErrorToStr(rc));
    } else {
        mvLog(MVLOG_INFO, "%s() XLinkBootRemote returned success %s\n", __func__, XLinkErrorToStr(rc));
          sleepForSeconds(DEVICE_APPEAR_TIMEOUT_ON_OPEN);
    }

    return parseXLinkError(rc);
}

static ncStatus_t getDevAttributes(struct _devicePrivate_t *d) {
    XLinkError_t rc = X_LINK_SUCCESS;
    CHECK_MUTEX_SUCCESS_RC(pthread_mutex_lock(&d->dev_stream_m), NC_ERROR);
    deviceCommand_t config = {0};
    config.type = DEVICE_GET_CAPABILITIES;
    rc = XLinkWriteData(d->device_mon_stream_id, (const uint8_t*)&config, sizeof(config));
    if (rc != X_LINK_SUCCESS) {
        mvLog(MVLOG_ERROR, "Failed to write data, rc: %s", XLinkErrorToStr(rc));
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_stream_m));
        return parseXLinkError(rc);
    }
    streamPacketDesc_t* packet = 0;
    rc = XLinkReadData(d->device_mon_stream_id, &packet);
    if (rc != X_LINK_SUCCESS || !packet) {
        mvLog(MVLOG_ERROR, "Failed to read data, rc: %s", XLinkErrorToStr(rc));
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_stream_m));
        return parseXLinkError(rc);
    }
    if(packet->length != sizeof(d->dev_attr)) {
        mvLog(MVLOG_ERROR, "Broken protocol. DevData can't be read\n");
        if (XLinkReleaseData(d->device_mon_stream_id) != X_LINK_SUCCESS) {
            mvLog(MVLOG_WARN, "Failed to release data\n");
        }
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_stream_m));
        return NC_ERROR;
    }
    d->dev_attr = *(deviceCapabilities_t*)packet->data;
    rc = XLinkReleaseData(d->device_mon_stream_id);
    CHECK_MUTEX_SUCCESS_RC(pthread_mutex_unlock(&d->dev_stream_m), NC_ERROR);
    if (rc != X_LINK_SUCCESS) {
        mvLog(MVLOG_WARN, "Failed to release data, rc: %s", XLinkErrorToStr(rc));
    }
    mvLog(MVLOG_INFO, "Device attributes\n");
    mvLog(MVLOG_INFO, "Device FW version: %x.%x.%x.%x\n", d->dev_attr.fw_version[0],
          d->dev_attr.fw_version[1], d->dev_attr.fw_version[2], d->dev_attr.fw_version[3]);
    mvLog(MVLOG_INFO, "Maximum graphs: %d\n", d->dev_attr.max_graphs);
    mvLog(MVLOG_INFO, "Maximum fifos: %d\n", d->dev_attr.max_fifos);
    mvLog(MVLOG_INFO, "Device memory capacity: %d\n", d->dev_attr.max_memory);
    return NC_OK;
}

static ncStatus_t getThermalStats(struct _devicePrivate_t *d){
    if (!d->thermal_stats){
        d->thermal_stats = calloc(THERMAL_THROTTLING_BUFFER_SIZE, 1);
        if (!d->thermal_stats)
            return NC_OUT_OF_MEMORY;
    }
    deviceCommand_t config;
    config.type = DEVICE_GET_THERMAL_STATS;
    CHECK_MUTEX_SUCCESS_RC(pthread_mutex_lock(&d->dev_stream_m), NC_ERROR);
    XLinkError_t rc = X_LINK_SUCCESS;
    rc = XLinkWriteData(d->device_mon_stream_id, (const uint8_t*)&config, sizeof(config));
    if (rc != X_LINK_SUCCESS) {
        mvLog(MVLOG_ERROR, "Failed to write data, rc: %s", XLinkErrorToStr(rc));
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_stream_m));
        return parseXLinkError(rc);
    }
    streamPacketDesc_t* packet = 0;
    rc = XLinkReadData(d->device_mon_stream_id, &packet);
    if (rc != X_LINK_SUCCESS || !packet) {
        mvLog(MVLOG_ERROR, "Failed to read data, rc: %s", XLinkErrorToStr(rc));
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_stream_m));
        return parseXLinkError(rc);
    }
    if( packet->length != THERMAL_THROTTLING_BUFFER_SIZE) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_stream_m));
        return NC_ERROR;
    }
    mvnc_memcpy(d->thermal_stats, THERMAL_THROTTLING_BUFFER_SIZE, packet->data, packet->length);
    rc = XLinkReleaseData(d->device_mon_stream_id);
    CHECK_MUTEX_SUCCESS_RC(pthread_mutex_unlock(&d->dev_stream_m), NC_ERROR);
    if (rc != X_LINK_SUCCESS) {
        mvLog(MVLOG_WARN,"Failed to release data, rc: %s", XLinkErrorToStr(rc));
    }
    return NC_OK;
}

static ncStatus_t deviceGetDeviceMemory(struct _devicePrivate_t *d,
                                        uint32_t * mem)
{
    deviceCommand_t config;
    config.type = DEVICE_GET_USED_MEMORY;
    CHECK_MUTEX_SUCCESS_RC(pthread_mutex_lock(&d->dev_stream_m), NC_ERROR);
    if (XLinkWriteData(d->device_mon_stream_id, (const uint8_t *) &config,
                       sizeof(config)) != 0) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_stream_m));
        return NC_ERROR;
    }
    streamPacketDesc_t *packet = 0;

    if (XLinkReadData(d->device_mon_stream_id, &packet) != 0 || !packet) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_stream_m));
        return NC_ERROR;
    }

    if (packet->length != (sizeof(uint32_t))) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_stream_m));
        return NC_ERROR;
    }
    mvnc_memcpy(mem, sizeof(uint32_t), packet->data, packet->length);
    XLinkReleaseData(d->device_mon_stream_id);
    CHECK_MUTEX_SUCCESS_RC(pthread_mutex_unlock(&d->dev_stream_m), NC_ERROR);
    return NC_OK;
}

static ncStatus_t deviceSetStdIO2XLink(struct _devicePrivate_t *d, uint32_t data)
{
    deviceCommand_t config;
    config.type = DEVICE_SET_STDIO_REDIRECT_XLINK;
    config.arg = data;
    CHECK_MUTEX_SUCCESS_RC(pthread_mutex_lock(&d->dev_stream_m), NC_ERROR);
    if (XLinkWriteData(d->device_mon_stream_id, (const uint8_t *) &config,
                       sizeof(config)) != 0) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_stream_m));
        return NC_ERROR;
    }
    CHECK_MUTEX_SUCCESS_RC(pthread_mutex_unlock(&d->dev_stream_m), NC_ERROR);
    return NC_OK;
}

#if (!defined(_WIN32) && !defined(_WIN64) && !defined(ANDROID))

#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>

static void fprintfsock( int s, const char* fmt, ... ) {
    char* buf = NULL;
    const char* ptext;
    int len;
    va_list args;
    if(fmt == NULL) {
        va_start( args, fmt);
        ptext = va_arg(args,const char*);
        len = va_arg(args, int);
        va_end(args);
    } else {
        va_start( args, fmt );
        len = vsnprintf( 0, 0, fmt, args ) + 1;
        buf = (char*) malloc(len);
        if (buf == NULL) {
            va_end (args);
            return;
        }
        va_start( args, fmt );
        vsnprintf( buf, len, fmt, args );
        va_end (args);
        ptext = buf;
    }

    if(s < 0) {
        if(write( 1, ptext, len) != len) {
            fprintf(stderr, "Error in fprintfsock: write failed\n");
        }
    } else {
        if(send( s, ptext, len, 0 ) < 0)
        {
            fprintf(stderr,"WARNING in fprintfsock: not all data has been sent\n");
        }
    }

    if(buf)
        free( buf );
}

static void* debugConsoleThreadReader(void* ctx) {
    struct _devicePrivate_t *d = (struct _devicePrivate_t *) ctx;
    streamId_t streamId = d->printf_over_xlink_stream_id;
    int connfd = d->printf_over_xlink_conn_fd;
    streamPacketDesc_t * packet;
    XLinkError_t xerr;

    fprintfsock(connfd, "XLinkConsole receiving loop begins\n");
    fprintfsock(connfd, "=========================================\n");
    while(1){
        // use 0 as the timeout to prevent trigger false reset
        xerr = XLinkReadDataWithTimeOut(streamId, &packet, 0);
        if(X_LINK_SUCCESS != xerr || packet == NULL)
            break;
        fprintfsock(connfd, NULL, packet->data, packet->length);
        XLinkReleaseData(streamId);
    }
    fprintfsock(connfd, "=========================================\n"
                        "Session closed (%d)\n", xerr);
    close(connfd);
    return NULL;
}

static void printfOverXLinkClose(struct _devicePrivate_t *d) {
    if(d->printf_over_xlink_stream_id != INVALID_STREAM_ID) {
        /* Tell device stop redirect STDIO to XLink Console */
        deviceSetStdIO2XLink(d, 0);
        XLinkCloseStream(d->printf_over_xlink_stream_id);
        d->printf_over_xlink_stream_id = INVALID_STREAM_ID;
    }

    if(d->printf_over_xlink_thr_valid) {
        pthread_cancel(d->printf_over_xlink_thr);
        d->printf_over_xlink_thr_valid = 0;
    }

    if(d->printf_over_xlink_conn_fd >= 0) {
        close(d->printf_over_xlink_conn_fd);
        d->printf_over_xlink_conn_fd = -1;
    }
}

// FIXME: update the function below to use mvLog instead of printf for consistency: #16773
static void printfOverXLinkOpen(struct _devicePrivate_t *d) {
    int linkId = d->xlink->linkId;
    const char * streamName = "console";
    streamId_t streamId = INVALID_STREAM_ID;
    char * cfg_use_xlink_printf = NULL;

    d->printf_over_xlink_stream_id = INVALID_STREAM_ID;
    d->printf_over_xlink_conn_fd = -1;
    d->printf_over_xlink_thr_valid = 0;

    /* export XLINK_PRINTF=1 to enable this feature */
    cfg_use_xlink_printf = getenv("XLINK_PRINTF");
    if(cfg_use_xlink_printf == NULL)
        return;
    if(strcmp(cfg_use_xlink_printf, "1") != 0)
        return;

    /* Tell device redirect STDIO to XLink Console */
    deviceSetStdIO2XLink(d, 1);

    streamId = XLinkOpenStream(linkId, streamName, 10*1024);
    if(streamId == INVALID_STREAM_ID) {
        fprintf(stderr,"ERROR in XLinkOpenStream: %s\n", streamName);
        return;
    }

    const char * servername = "localhost";
    struct hostent *server;
    struct sockaddr_in serv_addr;

    server = gethostbyname(servername);
    if (server == NULL) {
        fprintf(stderr,"ERROR in gethostbyname: %s\n", servername);
        return;
    }

    int portNum = 7788;
    int connfd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);

    bzero((char *) &serv_addr, sizeof(serv_addr));
    serv_addr.sin_family = AF_INET;
    bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length);
    serv_addr.sin_port = htons(portNum);

    /* Now connect to the server */
    if (connect(connfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) {
        printf("WARNNING: Cannot connect to XlinkPrintf debug console server, will print in current console instead\n");
        // even when no debug server, we still need drain possible debug information out of the XLink
        // or it will hang
        close(connfd);
        connfd = -1;
    }

    d->printf_over_xlink_stream_id = streamId;
    d->printf_over_xlink_conn_fd = connfd;
    if(pthread_create(&d->printf_over_xlink_thr, NULL, debugConsoleThreadReader, (void*) d)){
        fprintf(stderr,"ERROR in creating XlinkPrintf debug console reader thread!\n");
        printfOverXLinkClose (d);
    }else {
        d->printf_over_xlink_thr_valid = 1;
    }
}

#endif


static int findDevice(struct _devicePrivate_t *deviceHandle)
{

    struct _devicePrivate_t *d = devices;

    while (d) {
        if (d == deviceHandle)
            return 0;
        d = d->next;
    }

    return -1;
}

static int findGraph(struct _graphPrivate_t *graphHandle)
{
    struct _devicePrivate_t *d = devices;

    while (d) {
        struct _graphPrivate_t *g = d->graphs;
        while (g) {
            if (g == graphHandle)
                return 0;
            g = g->next;
        }
        d = d->next;
    }

    return -1;
}

// Defined here as it will be used twice
static int deallocateGraph(struct _graphPrivate_t *g)
{
    int found = 0;
    if (!g) {
        return -!found;
    }
    // Remove it from the list of the associated device
    if (g->dev->graphs == g) {
        g->dev->graphs = g->next;
        found = 1;
    } else {
        struct _graphPrivate_t *gp = g->dev->graphs;
        while (gp->next) {
            if (gp->next == g) {
                found = 1;
                gp->next = gp->next->next;
                break;
            }
            gp = gp->next;
        }
    }

    // Free it with all its data
    if (found) {
        free(g->aux_buffer);
    }
    g->state = NC_GRAPH_DEALLOCATED;
    return -!found;
}

static int findFifo(struct _fifoPrivate_t *f)
{
    if (!f || !f->dev)
        return 0;

    if (f->dev->fifos == f) {
        return 1;
    } else {
        struct _fifoPrivate_t *fp = f->dev->fifos;
        while (fp->next) {
            if (fp->next == f) {
                return 1;
            }
            fp = fp->next;
        }
    }
    return 0;
}

static int deallocateFifo(struct _fifoPrivate_t *f)
{
    int found = 0;
    if (!f) {
        return -!found;
    }
    // Remove it from the list of the associated device
    if (f->dev->fifos == f) {
        f->dev->fifos = f->next;
        found = 1;
    } else {
        struct _fifoPrivate_t *fp = f->dev->fifos;
        while (fp->next) {
            if (fp->next == f) {
                found = 1;
                fp->next = fp->next->next;
                break;
            }
            fp = fp->next;
        }
    }

    // Free it with all its data
    if (found) {
        //deallocate on device
        XLinkCloseStream(f->streamId);
        struct _userParamPrivate_t *temp;
        while (f->user_param_in) {
            temp = f->user_param_in;
            f->user_param_in = f->user_param_in->next;
            free(temp);
        }
        while (f->user_param_out) {
            temp = f->user_param_out;
            f->user_param_out = f->user_param_out->next;
            free(temp);
        }
    }
    f->state = NC_FIFO_DEALLOCATED;
    return -!found;
}

static ncStatus_t destroyDeviceHandle(struct ncDeviceHandle_t **deviceHandlePtr) {
    if (!deviceHandlePtr) {
        mvLog(MVLOG_ERROR, "Handle is NULL");
        return NC_INVALID_HANDLE;
    }
    if (!(*deviceHandlePtr)) {
        mvLog(MVLOG_INFO, "Handle already destroyed");
        return NC_OK;
    }

    mvLog(MVLOG_INFO, "Destroying device handler");

    struct _devicePrivate_t *d = (*deviceHandlePtr)->private_data;

    if(d->next) {
        mvLog(MVLOG_WARN, "Device could be in mvnc devices list");
    }

    free(d->thermal_stats);
    free(d->dev_addr);
    free(d->dev_addr_booted);

    free(d->dev_file);
    free(d->optimisation_list);

    free(d->xlink);

    free(d);
    (*deviceHandlePtr)->private_data = NULL;
    free((*deviceHandlePtr));
    *deviceHandlePtr = NULL;

    return NC_OK;
}


ncStatus_t ncDeviceClose(struct ncDeviceHandle_t **deviceHandlePtr) {
    int found = 0;
    XLinkError_t rc = X_LINK_SUCCESS;

    if (!deviceHandlePtr) {
        mvLog(MVLOG_ERROR, "Handle is NULL");
        return NC_INVALID_HANDLE;
    }
    if (!(*deviceHandlePtr)) {
        mvLog(MVLOG_INFO, "Handle already destroyed");
        return NC_OK;
    }

    struct _devicePrivate_t *d = (*deviceHandlePtr)->private_data;
    if (!d) {
        mvLog(MVLOG_ERROR, "Device has been destroyed");
        return NC_INVALID_HANDLE;
    }

    int wasConnectedToBooted = 0;
    if (d->dev_addr != NULL && d->dev_addr_booted != NULL &&
        strncmp(d->dev_addr, d->dev_addr_booted, NC_MAX_NAME_SIZE) == 0) {
        // PCIe device have same booted and unbooted addr
        if (d->protocol != X_LINK_PCIE)
            wasConnectedToBooted = 1;
    }

    GLOBAL_LOCK();
    if (findDevice(d)) {
        GLOBAL_UNLOCK();
        return NC_INVALID_PARAMETERS;
    }
    mvLog(MVLOG_INFO, "Removing device...");

    // Remove it from our list
    if (devices == d) {
        devices = d->next;
        found = 1;
    } else {
        struct _devicePrivate_t *dp = devices;
        while (dp->next) {
            if (dp->next == d) {
                found = 1;
                dp->next = dp->next->next;
                break;
            }
            dp = dp->next;
        }
    }
    d->next = NULL;

    if (!found) {
        GLOBAL_UNLOCK();
        return NC_INVALID_PARAMETERS;
    }
    // Deallocate all associated graphs
    CHECK_MUTEX_SUCCESS(pthread_mutex_lock(&d->dev_data_m));
    if (d->graphs) {
        mvLog(MVLOG_WARN,
              "Graphs on the device hasn't been destroyed! Graphs will be deallocated");
        while (deallocateGraph(d->graphs) != -1) {
            mvLog(MVLOG_INFO, "Graph was deallocated");
        }
    }
    // Deallocate all associated fifos
    if (d->fifos) {
        mvLog(MVLOG_WARN,
              "Fifos on the device hasn't been destroyed! Fifos will be deallocated");
        while (deallocateFifo(d->fifos) != -1) {
            mvLog(MVLOG_INFO, "Fifo was deallocated");
        }
    }

#if (!defined(_WIN32) && !defined(_WIN64) && !defined(ANDROID))
    printfOverXLinkClose(d);
#endif

#if !defined(NO_BOOT)
    watchdog_unregister_device(&d->watchdog_ctx);
#endif

    // Save all devices before reset
    deviceDesc_t in_deviceDesc = {
            .platform = X_LINK_ANY_PLATFORM,
            .protocol = d->protocol
    };
    deviceDesc_t beforeResetDevices[NC_MAX_DEVICES] = {{0}};
    unsigned int foundDevicesBeforeReset = 0;
    XLinkFindAllSuitableDevices(X_LINK_ANY_STATE, in_deviceDesc, beforeResetDevices,
                                NC_MAX_DEVICES, &foundDevicesBeforeReset);

        // #17801
#if !defined(NO_BOOT)
    if (d->graph_monitor_stream_id != INVALID_LINK_ID) {
        if (d->device_mon_stream_id != INVALID_LINK_ID) {
            rc = XLinkCloseStream(d->device_mon_stream_id);
            if (rc)
                mvLog(MVLOG_WARN,"Failed to close stream, rc: %s", XLinkErrorToStr(rc));
        }
        rc = XLinkCloseStream(d->graph_monitor_stream_id);
        if (rc)
            mvLog(MVLOG_WARN,"Failed to close stream, rc: %s", XLinkErrorToStr(rc));
    }
#endif
    // Reset device
    // In case when we open already booted device (or PCIE), just close connection to device
    rc = XLinkResetRemote(d->xlink->linkId);
    if (wasConnectedToBooted) {
        mvLog(MVLOG_INFO, "Only device handle will be released and link to device closed");
        if (rc)
            mvLog(MVLOG_WARN, "Failed to close link to device, rc: %s", XLinkErrorToStr(rc));
    } else {
        if (rc)
            mvLog(MVLOG_WARN, "Failed to reset, rc: %s", XLinkErrorToStr(rc));
    }

    d->state = NC_DEVICE_CLOSED;

    CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->graph_stream_m));
    CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->dev_stream_m));

    CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_data_m));
    CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(&d->dev_data_m));


    if (!wasConnectedToBooted && d->protocol != X_LINK_PCIE) {
        deviceDesc_t bootedDeviceDesc = {
                .protocol = d->protocol,
                .platform = X_LINK_ANY_PLATFORM
        };
        mv_strcpy(bootedDeviceDesc.name, XLINK_MAX_NAME_SIZE, d->dev_addr_booted);

        int booted_disappeared = 0;
        int unbooted_appeared = 0;

        //  Wait for unbooted device appear in usb list
        double waittm = timeInSeconds() + DEVICE_APPEAR_TIMEOUT_ON_CLOSE;

        deviceDesc_t afterResetDevices[NC_MAX_DEVICES] = {{0}};
        unsigned int foundDevicesAfterReset = 0;
        do {
            XLinkFindAllSuitableDevices(X_LINK_ANY_STATE, in_deviceDesc, afterResetDevices,
                                        NC_MAX_DEVICES, &foundDevicesAfterReset);
            if (foundDevicesAfterReset != foundDevicesBeforeReset) {
                continue;
            }

            deviceDesc_t deviceDesc = { 0 };

            rc = XLinkFindFirstSuitableDevice(X_LINK_BOOTED, bootedDeviceDesc, &deviceDesc);
            if (rc == X_LINK_SUCCESS) {
                continue;
            } else {
                booted_disappeared = 1;
            }
            int i, j;
            for (i = 0; i < foundDevicesAfterReset; ++i) {
                int found_in_before_reset_list = 0;
                for (j = 0; j < foundDevicesBeforeReset; ++j) {
                    if(strcmp(beforeResetDevices[i].name, afterResetDevices[j].name) == 0) {
                        found_in_before_reset_list = 1;
                    }
                }
                if (!found_in_before_reset_list) {
                    unbooted_appeared = 1;
                }
            }


        } while (!(booted_disappeared && unbooted_appeared) && timeInSeconds() < waittm);

        if (!booted_disappeared || !unbooted_appeared) {
            mvLog(MVLOG_ERROR, "Device didn't appear after reboot");
        }
    }

    ncStatus_t status = destroyDeviceHandle(deviceHandlePtr);
    GLOBAL_UNLOCK();
    if (status != NC_OK)
        mvLog(MVLOG_WARN, "Destroying device handle failed with error %s", ncStatusToStr(status));
    return status;
}

ncStatus_t ncGraphCreate(const char *name,
                         struct ncGraphHandle_t ** graphHandle)
{
    if ((!name) || (!graphHandle)) {
        mvLog(MVLOG_ERROR, "Some of the parameters are NULL");
        return NC_INVALID_PARAMETERS;
    }

    struct ncGraphHandle_t *gH = calloc(1, sizeof(*gH));
    struct _graphPrivate_t *g = calloc(1, sizeof(*g));

    if (!gH || !g) {
        free(g);
        free(gH);
        mvLog(MVLOG_ERROR, "Memory allocation failed");
        return NC_OUT_OF_MEMORY;
    }

    gH->private_data = g;
    mv_strncpy(g->name, NC_MAX_NAME_SIZE, name, NC_MAX_NAME_SIZE - 1);
    g->batch_size = 1;
    g->dev = NULL;
    g->executors_number = 1;
    g->started = 0;
    g->state = NC_GRAPH_CREATED;
    *graphHandle = gH;
    return NC_OK;
}

ncStatus_t trySendCommand(streamId_t graphMonStream, void* buffer, int size) {
    XLinkError_t rc = XLinkWriteData(graphMonStream, (uint8_t*)buffer, size);
    return parseXLinkError(rc);
}

ncStatus_t getGraphMonitorResponseValue(streamId_t graphMonStream, ncMvNCIErrorCode_t *value) {
    streamPacketDesc_t *ack = NULL;
    XLinkError_t rc = X_LINK_SUCCESS;
    rc = XLinkReadData(graphMonStream, &ack);
    if (rc) {
        mvLog(MVLOG_ERROR, "XLink error, rc: %s", XLinkErrorToStr(rc));
        return parseXLinkError(rc);
    }

    if (value == NULL)
        return NC_ERROR;

    *value = MVNCI_SUCCESS;
    if (ack) {
        *value = *((int*)ack->data);
    } else {
        mvLog(MVLOG_ERROR, "Error with stream packet");
        return NC_ERROR;
    }

    rc = XLinkReleaseData(graphMonStream);
    if (rc) {
        mvLog(MVLOG_ERROR, "XLink error, rc: %s", XLinkErrorToStr(rc));
    }

    return NC_OK;
}

ncStatus_t checkGraphMonitorResponse(streamId_t graphMonStream) {
    ncMvNCIErrorCode_t value = MVNCI_SUCCESS;
    int rc = getGraphMonitorResponseValue(graphMonStream, &value);

    if (rc) {
        return rc;
    }

    if (value != MVNCI_SUCCESS){
        mvLog(MVLOG_ERROR, "Graph monitor request returned error %d", value);
        return NC_MYRIAD_ERROR;
    }

    return NC_OK;
}

static void lockAllInferences() {
    struct _devicePrivate_t *d = devices;
    while (d) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_lock(&d->graph_stream_m));
        d = d->next;
    }
    return;
}

static void unlockAllInferences() {
    struct _devicePrivate_t *d = devices;
    while (d) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->graph_stream_m));
        d = d->next;
    }
    return;
}

ncStatus_t ncGraphAllocate(struct ncDeviceHandle_t * deviceHandle,
                           struct ncGraphHandle_t * graphHandle,
                           const void *graphBuffer,
                           unsigned int graphBufferLength,
                           const void *graphHeader,
                           unsigned int graphHeaderLength)
{
    CHECK_HANDLE_CORRECT(deviceHandle);
    CHECK_HANDLE_CORRECT(graphHandle);
    CHECK_HANDLE_CORRECT(graphHeader);
    CHECK_HANDLE_CORRECT(graphBuffer);

    ncStatus_t rc = NC_OK;
    XLinkError_t xl_error = X_LINK_SUCCESS;
    mvLog(MVLOG_INFO, "Starting Graph allocation sequence\n");



    if (graphHeaderLength > graphBufferLength) {
        mvLog(MVLOG_ERROR, "graphHeaderLength > graphBufferLength");
        return NC_INVALID_PARAMETERS;
    }

    static int graphIdCount = 0;
    struct _graphPrivate_t *g = graphHandle->private_data;

    struct _devicePrivate_t *d = devices;
    if (graphBufferLength > d->dev_attr.max_memory) {
        mvLog(MVLOG_ERROR, "The graph file is bigger than the device memory");
        return NC_OUT_OF_MEMORY;
    }

    GLOBAL_LOCK();
    while (d) {
        if (d == deviceHandle->private_data)
            break;
        d = d->next;
    }
    //TODO: review lists of devices and graphs internally.
    //TODO: check if the graph is not already on the device
    if (!d) {
        GLOBAL_UNLOCK();
        mvLog(MVLOG_ERROR, "Device not found!");
        return NC_INVALID_PARAMETERS;
    }
    GLOBAL_UNLOCK();

    lockAllInferences();
    g->id = graphIdCount++;
    streamId_t streamId;

    if (g->executors_number > d->dev_attr.max_executors) {
        mvLog(MVLOG_ERROR, "Executors number is greater than max allowed!");
        unlockAllInferences();
        return NC_INVALID_PARAMETERS;
    }

    graphCMDCommand_t cmd;
    cmd.type = GRAPH_VERIFY_CMD;
    snprintf(cmd.streamName, MAX_STREAM_NAME_LENGTH, "graphBuffer%d", g->id);
    streamId = XLinkOpenStream(d->xlink->linkId, cmd.streamName, graphBufferLength);
    CHECK_STREAM_ID(streamId, unlockAllInferences(), "can't open stream for graphBuffer transmission");

    cmd.id = g->id;
    cmd.executors_number = g->executors_number;

    rc = trySendCommand(d->graph_monitor_stream_id, &cmd, sizeof(cmd));
    if(rc != 0){
        mvLog(MVLOG_ERROR, "can't send graph allocation command");
        unlockAllInferences();
        return rc;
    }
    xl_error = XLinkWriteData(streamId, graphHeader, graphHeaderLength);
    if (xl_error) {
        mvLog(MVLOG_ERROR, "can't send graph header data to device, rc: %s", XLinkErrorToStr(xl_error));
        unlockAllInferences();
        return parseXLinkError(xl_error);
    }
    // for now simple status code used for graph header analysis result
    if ((rc = checkGraphMonitorResponse(d->graph_monitor_stream_id)) != 0) {
        mvLog(MVLOG_ERROR, "can't receive graph header verification response");
        unlockAllInferences();
        return rc;
    }

    // now sending whole graph with same header
    cmd.type = GRAPH_ALLOCATE_CMD;

    rc = trySendCommand(d->graph_monitor_stream_id, &cmd, sizeof(cmd));
    if(rc != 0){
        mvLog(MVLOG_ERROR, "can't send graph allocation command");
        unlockAllInferences();
        return rc;
    }
    xl_error = XLinkWriteData(streamId, graphBuffer, graphBufferLength);
    if (xl_error) {
        mvLog(MVLOG_ERROR, "can't send graph data to device, rc: %s", XLinkErrorToStr(xl_error));
        unlockAllInferences();
        return parseXLinkError(xl_error);
    }
    mvLog(MVLOG_INFO, "Sent graph");
    streamPacketDesc_t * tensorDescIn = 0;
    streamPacketDesc_t * tensorDescOut = 0;
    streamPacketDesc_t * nstages = 0;


    xl_error = XLinkReadData(streamId, &tensorDescIn);
    if (xl_error) {
        mvLog(MVLOG_ERROR, "Can't read input tensor descriptors of the graph, rc: %s", XLinkErrorToStr(xl_error));
        unlockAllInferences();
        return parseXLinkError(xl_error);
    }
    xl_error = XLinkReadData(streamId, &tensorDescOut);
    if (xl_error) {
        mvLog(MVLOG_ERROR, "Can't read output tensor descriptors of the graph, rc: %s", XLinkErrorToStr(xl_error));
        unlockAllInferences();
        return parseXLinkError(xl_error);
    }
    xl_error = XLinkReadData(streamId, &nstages);
    if (xl_error || nstages == NULL) {
        mvLog(MVLOG_WARN, "Can't read nstages, rc: %s", XLinkErrorToStr(xl_error));
        unlockAllInferences();
        return parseXLinkError(xl_error);
    }
    // for now, support only count 1
    if(!tensorDescIn ||
        tensorDescIn->length % sizeof(struct tensorDescriptor_t) ||
        tensorDescIn->length / sizeof(struct tensorDescriptor_t) > 1) {
        mvLog(MVLOG_ERROR, "Input tensor descriptors of the graph are invalid\n");
        if (tensorDescIn)
            mvLog(MVLOG_ERROR, "Received data from graph %d\n", *(int*)tensorDescIn->data);
        rc = NC_MYRIAD_ERROR;
    }
    // for now, support only count 1
    if(!tensorDescOut ||
        tensorDescOut->length % sizeof(struct tensorDescriptor_t) ||
        tensorDescOut->length / sizeof(struct tensorDescriptor_t) > 1) {
        mvLog(MVLOG_ERROR, "Output tensor descriptors of the graph are invalid\n");
        rc = NC_MYRIAD_ERROR;
    }
    if (rc == NC_OK){
        g->input_count = tensorDescIn->length / sizeof(struct tensorDescriptor_t);
        mvnc_memcpy(&g->input_tensor_desc, sizeof(struct tensorDescriptor_t),
                 tensorDescIn->data, sizeof(struct tensorDescriptor_t));
        g->output_count = tensorDescOut->length / sizeof(struct tensorDescriptor_t);
        mvnc_memcpy(&g->output_tensor_desc, sizeof(struct tensorDescriptor_t),
                 tensorDescOut->data, sizeof(struct tensorDescriptor_t));
        g->nstages = *(uint32_t*)nstages->data;
        g->batch_size = g->input_tensor_desc.n;
        g->timingsCount = g->nstages + 2;       // For time_receive timing and thread execution
    }

    xl_error = XLinkReleaseData(streamId);
    if (xl_error)
        mvLog(MVLOG_WARN, "Can't release data, rc: %s", XLinkErrorToStr(xl_error));

    xl_error = XLinkReleaseData(streamId);
    if (xl_error)
        mvLog(MVLOG_WARN, "Can't release data, rc: %s", XLinkErrorToStr(xl_error));

    xl_error = XLinkReleaseData(streamId);
    if (xl_error)
        mvLog(MVLOG_WARN, "Can't release data, rc: %s", XLinkErrorToStr(xl_error));

    g->graph_stream_id = streamId;
    if(checkGraphMonitorResponse(d->graph_monitor_stream_id)) {
        mvLog(MVLOG_ERROR, "The device didn't accept the graph\n");
        unlockAllInferences();
        return NC_ERROR;
    }
    if (rc) {
        unlockAllInferences();
        return rc;
    }

    ncMvNCIErrorCode_t allocation_error = MVNCI_SUCCESS;
    cmd.type = GRAPH_ALLOCATION_VERIFY_CMD;

    rc = trySendCommand(d->graph_monitor_stream_id, &cmd, sizeof(cmd));
    if(rc != 0){
        mvLog(MVLOG_ERROR, "can't send graph verification command");
        unlockAllInferences();
        return rc;
    }
    if (getGraphMonitorResponseValue(d->graph_monitor_stream_id, &allocation_error) != 0) {
        mvLog(MVLOG_ERROR, "Can't receive graph allocation verification response");
        unlockAllInferences();
        return NC_ERROR;
    }
    if (allocation_error != MVNCI_SUCCESS) {
        if (allocation_error == MVNCI_OUT_OF_MEMORY) {
            mvLog(MVLOG_ERROR, "Not enough memory to allocate intermediate tensors on remote device");
            unlockAllInferences();
            return NC_OUT_OF_MEMORY;
        } else {
            mvLog(MVLOG_ERROR, "Graph allocation caused an error %s on the device",
                  ncMvNCIErrorCodeToStr(allocation_error));
            unlockAllInferences();
            return NC_MYRIAD_ERROR;
        }
    }

    // aux_buffer
    g->aux_buffer = calloc(1, 224 + g->timingsCount * sizeof(*g->time_taken));
    if (!g->aux_buffer) {
        unlockAllInferences();
        return NC_OUT_OF_MEMORY;
    }
    // output_data

    g->debug_buffer = g->aux_buffer;
    g->time_taken = (float *) (g->aux_buffer + 120);
    unlockAllInferences();

    GLOBAL_LOCK();
    g->dev = d;

    if (d->graphs)
        g->next = d->graphs;
    d->graphs = g;
    g->state = NC_GRAPH_ALLOCATED;
    GLOBAL_UNLOCK();
    mvLog(MVLOG_INFO, "Graph allocation completed successfully\n");

    return NC_OK;
}

ncStatus_t ncGraphDestroy(struct ncGraphHandle_t ** graphHandle)
{
    CHECK_HANDLE_CORRECT(graphHandle);

    ncStatus_t rc = NC_OK;
    struct ncGraphHandle_t *gh = *graphHandle;
    if (!gh) {
        mvLog(MVLOG_INFO, "handle is already destroyed");
        return NC_OK;
    }
    struct _graphPrivate_t *g = gh->private_data;
    CHECK_HANDLE_CORRECT_WINFO(g, MVLOG_ERROR, "Graph handle is corrupt or has been destroyed");

    if (g->state == NC_GRAPH_CREATED || g->state == NC_GRAPH_DEALLOCATED) {
        free(g);
        gh->private_data = NULL;
        free(gh);
        *graphHandle = NULL;
        return NC_OK;
    }
    GLOBAL_LOCK();
    if (findGraph(g)) {
        GLOBAL_UNLOCK();
        mvLog(MVLOG_ERROR, "This graph is corrupt or has been destroyed");
        return NC_INVALID_HANDLE;
    }

    GLOBAL_UNLOCK();
    struct _devicePrivate_t *d = (gh->private_data)->dev;

    graphCMDCommand_t cmd;
    cmd.type = GRAPH_DEALLOCATE_CMD;
    cmd.id = g->id;
    CHECK_MUTEX_SUCCESS(pthread_mutex_lock(&d->graph_stream_m));
    rc = trySendCommand(d->graph_monitor_stream_id, &cmd, sizeof(cmd));
    if(rc != 0){
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->graph_stream_m));
        return rc;
    }
    if (checkGraphMonitorResponse(d->graph_monitor_stream_id)) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->graph_stream_m));
        return NC_ERROR;
    }
    XLinkCloseStream(g->graph_stream_id);
    CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->graph_stream_m));
    CHECK_MUTEX_SUCCESS(pthread_mutex_lock(&d->dev_data_m));
    if (deallocateGraph(gh->private_data)) {
        mvLog(MVLOG_ERROR, "This graph has already been destroyed");
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_data_m));
        return NC_INVALID_PARAMETERS;
    }
    CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_data_m));
    free(g);
    gh->private_data = NULL;
    free(gh);
    *graphHandle = NULL;
    return NC_OK;
}

ncStatus_t ncGraphSetOption(struct ncGraphHandle_t * graphHandle,
                            ncGraphOption_t option, const void *data,
                            unsigned int dataLength)
{
    CHECK_HANDLE_CORRECT(graphHandle);
    CHECK_HANDLE_CORRECT_WINFO(graphHandle->private_data, MVLOG_ERROR, "graphHandle has been destroyed");
    if (!data) {
        mvLog(MVLOG_ERROR, "Some of the parameters are NULL");
        return NC_INVALID_PARAMETERS;
    }
    if (option < GRAPH_OPTION_BASE ||
        option > (GRAPH_OPTION_BASE + OPTION_CLASS_SIZE * NC_OP_ACCESS_LAST)) {
        mvLog(MVLOG_ERROR, "Option %d is invalid", option);
        return NC_INVALID_PARAMETERS;
    }
    if (option >= GRAPH_OPTION_BASE &&
        option <= (GRAPH_OPTION_BASE + OPTION_CLASS_SIZE)) {
        mvLog(MVLOG_ERROR, "Option %d is read only", option);
        return NC_UNAUTHORIZED;
    }
    struct _graphPrivate_t *g = graphHandle->private_data;

    GLOBAL_LOCK();
    if (option == NC_RW_GRAPH_EXECUTORS_NUM && g->state != NC_GRAPH_CREATED) {
        mvLog(MVLOG_ERROR,
              "This graph has already been alocated - cannot set option");
        GLOBAL_UNLOCK();
        return NC_UNAUTHORIZED;
    }
    if (option != NC_RW_GRAPH_EXECUTORS_NUM && g->state == NC_GRAPH_CREATED) {
        mvLog(MVLOG_ERROR,
              "This graph hasn't been allocated - cannot set option");
        GLOBAL_UNLOCK();
        return NC_UNAUTHORIZED;
    }
    if (option != NC_RW_GRAPH_EXECUTORS_NUM && findGraph(g)) {
        mvLog(MVLOG_ERROR, "This graph is corrupt or has been destroyed");
        GLOBAL_UNLOCK();
        return NC_INVALID_HANDLE;
    }
    GLOBAL_UNLOCK();

    ncOptionAccess_t opAccess = getOptionAccess(option, GRAPH_OPTION_BASE);
    if(opAccess == NC_OP_ACCESS_READ_ONLY) {
        mvLog(MVLOG_ERROR, "Option is read-only");
        return NC_UNAUTHORIZED;
    }

    if (opAccess != NC_OP_ACCESS_READ_WRITE) {
        mvLog(MVLOG_ERROR, "There is no such option");
        return NC_INVALID_PARAMETERS;
    }

    switch (option) {
    case NC_RW_GRAPH_EXECUTORS_NUM:
        {
            if (dataLength < sizeof(int)) {
                mvLog(MVLOG_ERROR, "The dataLength is smaller that required %zu",
                      sizeof(int));
                return NC_INVALID_DATA_LENGTH;
            }

            if (g->state != NC_GRAPH_CREATED) {
                mvLog(MVLOG_ERROR, "Can't set NCE number after graph allocation");
                return NC_UNAUTHORIZED;
            }
            g->executors_number = *(int *) data;
            break;
        }
    default:
        mvLog(MVLOG_ERROR, "There is no such option");
        return NC_INVALID_PARAMETERS;
    }
    return NC_OK;
}

static ncStatus_t getGraphOption(struct _graphPrivate_t *g,
                                       ncGraphOption_t option,
                                       void *data, unsigned int *dataLength)
{
    if ((option == NC_RO_GRAPH_INPUT_COUNT ||
         option == NC_RO_GRAPH_OUTPUT_COUNT ||
         option == NC_RW_GRAPH_EXECUTORS_NUM) && *dataLength < sizeof(int)) {
        mvLog(MVLOG_ERROR,
              "data length of data (%d) is smaller that required (%zu)!\n",
              *dataLength, sizeof(int));
        *dataLength = sizeof(int);
        return NC_INVALID_DATA_LENGTH;
    }

    ncStatus_t rc = NC_OK;
    graphCommonCommand_t cmd;
    streamPacketDesc_t* pack = 0;

    switch (option) {
    case NC_RO_GRAPH_INPUT_COUNT:
        *(int *) data = g->input_count;
        *dataLength = sizeof(int);
        break;
    case NC_RO_GRAPH_OUTPUT_COUNT:
        *(int *) data = g->output_count;
        *dataLength = sizeof(int);
        break;
    case NC_RO_GRAPH_TIME_TAKEN_ARRAY_SIZE:
        *(int *) data = sizeof(float) * g->timingsCount;
        *dataLength = sizeof(int);
        break;
    case NC_RO_GRAPH_TIME_TAKEN:
        CHECK_HANDLE_CORRECT(g->dev);
        if (*dataLength < sizeof(float) * g->timingsCount) {
            mvLog(MVLOG_ERROR,
                  "data length of output buffer (%d) is smaller that required (%zu)!\n",
                  *dataLength, sizeof(float) * g->timingsCount);
            *dataLength = sizeof(float) * g->timingsCount;
            return NC_INVALID_DATA_LENGTH;
        }
        cmd.id = g->id;
        cmd.type = GRAPH_GET_TIMING_DATA;
        CHECK_MUTEX_SUCCESS_RC(pthread_mutex_lock(&g->dev->graph_stream_m), NC_ERROR);
        rc = trySendCommand(g->dev->graph_monitor_stream_id, &cmd, sizeof(cmd));
        if(rc != 0){
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
            return rc;
        }
        if (XLinkReadData(g->dev->graph_monitor_stream_id, &pack) || !pack) {
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
            return NC_ERROR;
        }
        if (pack->length != sizeof(float) * g->timingsCount) {
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
            XLinkReleaseData(g->dev->graph_monitor_stream_id);
            return NC_ERROR;
        }
        //Need to copy data before we check the response, since checkGraphMonitorResponse
        //calls releaseData
        mvnc_memcpy((float *) data, *dataLength, pack->data, pack->length);
        XLinkReleaseData(g->dev->graph_monitor_stream_id);

        if (checkGraphMonitorResponse(g->dev->graph_monitor_stream_id)) {
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
            return NC_ERROR;
        }

        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
        *dataLength = sizeof(float) * g->timingsCount;
        break;
    case NC_RO_GRAPH_DEBUG_INFO:
        CHECK_HANDLE_CORRECT(g->dev);
        if (*dataLength < NC_DEBUG_BUFFER_SIZE) {
            mvLog(MVLOG_ERROR,
                  "data length of output buffer (%d) is smaller that required (%d)!\n",
                  *dataLength, NC_DEBUG_BUFFER_SIZE);
            *dataLength = NC_DEBUG_BUFFER_SIZE;
            return NC_INVALID_DATA_LENGTH;
        }

        cmd.type = GRAPH_GET_DEBUG_DATA;
        cmd.id = g->id;
        CHECK_MUTEX_SUCCESS_RC(pthread_mutex_lock(&g->dev->graph_stream_m), NC_ERROR);
        rc = trySendCommand(g->dev->graph_monitor_stream_id, &cmd, sizeof(cmd));
        if(rc != 0){
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
            return rc;
        }

        if (XLinkReadData(g->dev->graph_monitor_stream_id, &pack) || !pack) {
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
            return NC_ERROR;
        }

        if (pack->length != NC_DEBUG_BUFFER_SIZE) {
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
            XLinkReleaseData(g->dev->graph_monitor_stream_id);
            return NC_ERROR;
        }

        mvnc_memcpy((char *) data, *dataLength, pack->data, pack->length);
        XLinkReleaseData(g->dev->graph_monitor_stream_id);
        if (checkGraphMonitorResponse(g->dev->graph_monitor_stream_id)) {
            CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
            return NC_ERROR;
        }
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));

        *dataLength = NC_DEBUG_BUFFER_SIZE;
        break;
    case NC_RO_GRAPH_INPUT_TENSOR_DESCRIPTORS:{
            unsigned int size =
                sizeof(struct ncTensorDescriptor_t) * g->input_count;
            if (*dataLength < size) {
                mvLog(MVLOG_ERROR,
                      "data length of output buffer (%d) is smaller that required (%d)!\n",
                      *dataLength, size);
                *dataLength = size;
                return NC_INVALID_DATA_LENGTH;
            }
            mvnc_memcpy((struct ncTensorDescriptor_t *) data, *dataLength,
                       &g->input_tensor_desc, size);
            *dataLength = size;
            break;
        }
    case NC_RO_GRAPH_OUTPUT_TENSOR_DESCRIPTORS:{
            unsigned int size =
                sizeof(struct ncTensorDescriptor_t) * g->output_count;
            if (*dataLength < size) {
                mvLog(MVLOG_ERROR,
                      "data length of output buffer (%d) is smaller that required (%d)!\n",
                      *dataLength, size);
                *dataLength = size;
                return NC_INVALID_DATA_LENGTH;
            }
            mvnc_memcpy((struct ncTensorDescriptor_t *) data, *dataLength,
                       &g->output_tensor_desc, size);
            *dataLength = size;
            break;
        }
    case NC_RO_GRAPH_VERSION:{
            unsigned int size = sizeof(g->blob_version);
            if (*dataLength < size) {
                mvLog(MVLOG_ERROR,
                      "data length of output buffer (%d) is smaller that required (%d)!\n",
                      *dataLength, size);
                *dataLength = size;
                return NC_INVALID_DATA_LENGTH;
            }
            mvnc_memcpy((int *) data, *dataLength, g->blob_version, size);
            *dataLength = size;
            break;
        }
    case NC_RW_GRAPH_EXECUTORS_NUM:{
        int size = sizeof(int);
        if (*dataLength < size) {
            mvLog(MVLOG_ERROR,
                  "data length of data (%d) is smaller that required (%d)!\n",
                  *dataLength, size);
            *dataLength = size;
            return NC_INVALID_DATA_LENGTH;
        }
        *(int *) data = g->executors_number;
        *dataLength = size;
        break;
        }
    default:
        mvLog(MVLOG_ERROR, "There is no such option");
        return NC_INVALID_PARAMETERS;
    }
    return NC_OK;
}

ncStatus_t ncGraphGetOption(struct ncGraphHandle_t * graphHandle,
                            ncGraphOption_t option, void *data, unsigned int *dataLength)
{
    CHECK_HANDLE_CORRECT(graphHandle);
    CHECK_HANDLE_CORRECT_WINFO(graphHandle->private_data, MVLOG_ERROR, "graphHandle has been destroyed");

    if (!dataLength || (*dataLength != 0 && !data)) {
        mvLog(MVLOG_ERROR, "Some of the parameters are NULL");
        return NC_INVALID_PARAMETERS;
    }

    if (option < GRAPH_OPTION_BASE ||
        option > (GRAPH_OPTION_BASE + OPTION_CLASS_SIZE * NC_OP_ACCESS_LAST)) {
        mvLog(MVLOG_ERROR, "Option %d is invalid", option);
        return NC_INVALID_PARAMETERS;
    }

    struct _graphPrivate_t *g = graphHandle->private_data;
    CHECK_HANDLE_CORRECT(g);

    GLOBAL_LOCK();
    if (option != NC_RW_GRAPH_EXECUTORS_NUM && g->state == NC_GRAPH_CREATED) {
        mvLog(MVLOG_ERROR, "This graph hasn't been allocated");
        GLOBAL_UNLOCK();
        return NC_NOT_ALLOCATED;
    }
    ncOptionAccess_t opAccess = getOptionAccess(option, GRAPH_OPTION_BASE);
    if (opAccess != NC_OP_ACCESS_READ_ONLY &&
        opAccess != NC_OP_ACCESS_READ_WRITE) {
        mvLog(MVLOG_ERROR, "There is no such option");
        return NC_INVALID_PARAMETERS;
    }

    GLOBAL_UNLOCK();

    return getGraphOption(g, option, data, dataLength);
}

ncStatus_t ncGlobalSetOption(ncGlobalOption_t option, const void *data,
                             unsigned int dataLength)
{
    if (!data) {
        mvLog(MVLOG_ERROR, "Some of the parameters are NULL");
        return NC_INVALID_PARAMETERS;
    }

    switch (option) {
    case NC_RW_LOG_LEVEL:
        {
            mvLog_t log_level = *(mvLog_t *) data;
            if (log_level >= MVLOG_LAST || log_level < 0) {
                mvLog(MVLOG_ERROR, "log_level value is invalid %d\n",
                      log_level);
                return NC_INVALID_PARAMETERS;
            }
            mvLogLevelSet(*(mvLog_t *) data);
            mvLogDefaultLevelSet(*(mvLog_t *) data);    //Allow turning off warnings and errors
        }
        break;
    case NC_RO_API_VERSION:
        mvLog(MVLOG_ERROR, "API version is read-only");
        return NC_UNAUTHORIZED;
    case NC_RW_RESET_ALL:
        if (!initialized)
            reset_all = *(int*)data;
        break;
    case NC_RW_COMMON_TIMEOUT_MSEC: {
        int gTimeout = *(int *) data;
        XLinkError_t rc = XLinkSetCommonTimeOutMsec(gTimeout);
        if (rc) {
            mvLog(MVLOG_ERROR, "Set global common timeout failed, rc = %s\n", XLinkErrorToStr(rc));
            return NC_INVALID_PARAMETERS;
        }
        break;
    }
    case NC_RW_DEVICE_OPEN_TIMEOUT_MSEC: {
        int gTimeout = *(int *) data;
        XLinkError_t rc = XLinkSetDeviceOpenTimeOutMsec(gTimeout);
        if (rc) {
            mvLog(MVLOG_ERROR, "Set global open device timeout failed, rc = %s\n", XLinkErrorToStr(rc));
            return NC_INVALID_PARAMETERS;
        }
        break;
    }
    default:
        mvLog(MVLOG_ERROR, "No such option");
        return NC_INVALID_PARAMETERS;
    }

    return NC_OK;
}

ncStatus_t ncGlobalGetOption(ncGlobalOption_t option, void *data, unsigned int *dataLength)
{
    if (!data || !dataLength) {
        mvLog(MVLOG_ERROR, "Some of the parameters are NULL");
        return NC_INVALID_PARAMETERS;
    }
    switch (option) {
    case NC_RW_LOG_LEVEL:
        *(int *) data = mvLogLevel_ncAPI;
        *dataLength = sizeof(mvLogLevel_ncAPI);
        break;
    case NC_RO_API_VERSION:
        return NC_UNSUPPORTED_FEATURE;
        break;
    case NC_RW_RESET_ALL:
        *(int*)data = reset_all;
        *dataLength = sizeof(reset_all);
        break;
    default:
        mvLog(MVLOG_ERROR, "No such option");
        return NC_INVALID_PARAMETERS;
    }

    return NC_OK;
}

static ncStatus_t getDeviceOption(struct _devicePrivate_t *d,
                                        ncDeviceOption_t option,
                                        void *data, unsigned int *dataLength)
{
    ncStatus_t rc = NC_OK;

    switch (option) {
    case NC_RO_DEVICE_THERMAL_STATS:
        if (*dataLength < NC_THERMAL_BUFFER_SIZE) {
            mvLog(MVLOG_ERROR,
                  "data length of output buffer (%d) is smaller that required (%d)!\n",
                  *dataLength, NC_THERMAL_BUFFER_SIZE);
            *dataLength = NC_THERMAL_BUFFER_SIZE;
            return NC_INVALID_DATA_LENGTH;
        }
        rc = getThermalStats(d);
        if (rc) {
            return rc;
        }
        mvnc_memcpy((float *) data, *dataLength, &d->thermal_stats[1], NC_THERMAL_BUFFER_SIZE);
        *dataLength = NC_THERMAL_BUFFER_SIZE;
        break;
    case NC_RO_DEVICE_THERMAL_THROTTLING_LEVEL:
        rc = getThermalStats(d);
        if (rc) {
            return rc;
        }
        d->throttle_happened = d->thermal_stats[0];
        *(int *) data = d->throttle_happened;
        *dataLength = sizeof(int);
        break;
    case NC_RO_DEVICE_MEMORY_SIZE:
        *(int *) data = d->dev_attr.max_memory;
        *dataLength = sizeof(int);
        break;
    case NC_RO_DEVICE_MAX_GRAPH_NUM:
        *(int *) data = d->dev_attr.max_graphs;
        *dataLength = sizeof(int);
        break;
    case NC_RO_DEVICE_NAME:
        if (*dataLength < strlen(d->dev_addr) + 1) {
            mvLog(MVLOG_ERROR,
                  "data length of output buffer (%d) is smaller that required (%zu)!\n",
                  *dataLength, strlen(d->dev_addr) + 1);
            *dataLength = strlen(d->dev_addr) + 1;
            return NC_INVALID_DATA_LENGTH;
        }
        *dataLength = strlen(d->dev_addr) + 1;
        mv_strncpy((char *) data, *dataLength, d->dev_addr, *dataLength - 1);
        break;
    case NC_RO_DEVICE_PLATFORM:
        if (d->dev_attr.fw_version[1] == 0x2480){
            *(ncDevicePlatform_t *) data = NC_MYRIAD_X;
        } else if (d->dev_attr.fw_version[1] == 0x2450) {
            *(ncDevicePlatform_t *) data = NC_MYRIAD_2;
        } else {
            *(ncDevicePlatform_t *) data = NC_ANY_PLATFORM;
        }
        *dataLength = sizeof(ncDevicePlatform_t);
        break;
    case NC_RO_DEVICE_PROTOCOL:
        *(ncDeviceProtocol_t *) data = convertProtocolToNC(d->protocol);
        *dataLength = sizeof(ncDeviceProtocol_t);
        break;
    case NC_RO_DEVICE_CURRENT_MEMORY_USED:{
            uint32_t mem;
            if (deviceGetDeviceMemory(d, &mem)) {
                rc = NC_ERROR;
                break;
            }
            *(int *) data = mem;
            *dataLength = sizeof(int);
            break;
        }
    default:
        mvLog(MVLOG_ERROR, "No such option");
        return NC_INVALID_PARAMETERS;
    }
    return rc;
}

static ncStatus_t setDevicePowerConfig(struct _devicePrivate_t *d,
                                        ncDeviceOption_t option,
                                        const void *data, unsigned int dataLength){
    XLinkError_t rc = X_LINK_SUCCESS;
    deviceCommand_t config;

    if (option != NC_RW_DEVICE_POWER_CONFIG_RESET && option != NC_RW_DEVICE_POWER_CONFIG) {
        mvLog(MVLOG_ERROR, "No such option");
        return NC_INVALID_PARAMETERS;
    }

    config.type = (option == NC_RW_DEVICE_POWER_CONFIG ? DEVICE_SET_POWER_CONFIG : DEVICE_RESET_POWER_CONFIG);
    config.arg = *(uint32_t*)data;

    rc = XLinkWriteData(d->device_mon_stream_id, (const uint8_t *)&config, sizeof(config));

    if (rc != X_LINK_SUCCESS)
    {
        mvLog(MVLOG_ERROR, "Failed to write data, rc: %s", XLinkErrorToStr(rc));
        return parseXLinkError(rc);
    }

    return NC_OK;
}

ncStatus_t ncDeviceSetOption(struct ncDeviceHandle_t *deviceHandle,
                            ncDeviceOption_t option,
                            const void *data, unsigned int dataLength){
    ncStatus_t rc = NC_OK;
    if (!deviceHandle || !data){
        mvLog(MVLOG_ERROR, "Some of the parameters are NULL");
        return NC_INVALID_PARAMETERS;
    }
    if (dataLength != sizeof(int) && dataLength != sizeof(void*)){
        mvLog(MVLOG_ERROR, "The dataLength must be %zu or %zu", sizeof(int), sizeof(void*));
        return NC_INVALID_PARAMETERS;
    }

    if (option < DEVICE_OPTION_BASE ||
        option > (DEVICE_OPTION_BASE + OPTION_CLASS_SIZE * NC_OP_ACCESS_LAST)) {
        mvLog(MVLOG_ERROR, "Option %d is invalid", option);
        return NC_INVALID_PARAMETERS;
    }

    ncOptionAccess_t opAccess = getOptionAccess(option, DEVICE_OPTION_BASE);
    if(opAccess == NC_OP_ACCESS_READ_ONLY) {
        mvLog(MVLOG_ERROR, "Option is read-only");
        return NC_UNAUTHORIZED;
    }

    struct _devicePrivate_t *d = deviceHandle->private_data;
    GLOBAL_LOCK();

    if (findDevice(d)) {
        mvLog(MVLOG_ERROR,
              "This device handle is corrupt or has been destroyed");
        GLOBAL_UNLOCK();

        return NC_INVALID_HANDLE;
    }

    if (opAccess != NC_OP_ACCESS_READ_WRITE) {
        mvLog(MVLOG_ERROR, "There is no such option");
        return NC_INVALID_PARAMETERS;
    }

    switch (option) {
        case NC_RW_DEVICE_POWER_CONFIG:
        case NC_RW_DEVICE_POWER_CONFIG_RESET:
        {
            rc = setDevicePowerConfig(d, option, data, dataLength);
            break;
        }
        default:
            rc = NC_INVALID_PARAMETERS;
            mvLog(MVLOG_ERROR, "There is no such option");
    }

    GLOBAL_UNLOCK();
    return rc;
}

//static options can be read before device is open
static int isDeviceStaticOption(int option)
{
    switch (option) {
    case NC_RO_DEVICE_NAME:
        return 1;
    default:
        return 0;
    }
}

ncStatus_t ncDeviceGetOption(struct ncDeviceHandle_t * deviceHandle,
        ncDeviceOption_t option, void *data, unsigned int *dataLength)
{
    CHECK_HANDLE_CORRECT(deviceHandle);
    ncStatus_t rc;

    if (!dataLength || (*dataLength != 0 && !data)) {
        mvLog(MVLOG_ERROR, "Some of the parameters are NULL");
        return NC_INVALID_PARAMETERS;
    }

    if (option < DEVICE_OPTION_BASE ||
        option > (DEVICE_OPTION_BASE + OPTION_CLASS_SIZE * NC_OP_ACCESS_LAST)) {
        mvLog(MVLOG_ERROR, "Option %d is invalid", option);
        return NC_INVALID_PARAMETERS;
    }

    struct _devicePrivate_t *d = deviceHandle->private_data;

    GLOBAL_LOCK();
    if (!isDeviceStaticOption(option) && d->state != NC_DEVICE_OPENED) {
        mvLog(MVLOG_ERROR, "This device hasn't been opened");
        GLOBAL_UNLOCK();
        return NC_UNAUTHORIZED;
    }

    if (!isDeviceStaticOption(option)) {
        if (findDevice(d)) {
            mvLog(MVLOG_ERROR,
                  "This device handle is corrupt or has been destroyed");
            GLOBAL_UNLOCK();
            return NC_INVALID_HANDLE;
        }
    }

    ncOptionAccess_t opAccess = getOptionAccess(option, DEVICE_OPTION_BASE);
    if (opAccess != NC_OP_ACCESS_READ_ONLY &&
       opAccess != NC_OP_ACCESS_READ_WRITE) {
        mvLog(MVLOG_ERROR, "There is no such option");
        return NC_INVALID_PARAMETERS;
    }

    rc = getDeviceOption(d, option, data, dataLength);
    GLOBAL_UNLOCK();

    return rc;
}

static int fifoWriteAccess(struct _fifoPrivate_t *fifoHandle)
{
    if (fifoHandle->type == NC_FIFO_HOST_WO) {
        return 1;
    }
    return 0;
}

static int fifoReadAccess(struct _fifoPrivate_t *fifoHandle)
{
    if (fifoHandle->type == NC_FIFO_HOST_RO) {
        return 1;
    }
    return 0;
}

ncStatus_t ncFifoCreate(const char *name, ncFifoType_t type,
                        struct ncFifoHandle_t ** fifoHandle)
{
    mvLog(MVLOG_INFO, "Init fifo");
    CHECK_HANDLE_CORRECT(fifoHandle);
    CHECK_HANDLE_CORRECT(name);

    if (type != NC_FIFO_HOST_RO && type != NC_FIFO_HOST_WO) {
        mvLog(MVLOG_ERROR, "Fifo typo not supported!");
        return NC_UNSUPPORTED_FEATURE;
    }

    static int fifoIdCounter = 0;
    *fifoHandle = (struct ncFifoHandle_t *) malloc(sizeof(struct ncFifoHandle_t));
    if (!(*fifoHandle)) {
        mvLog(MVLOG_ERROR, "Memory allocation failed");
        return NC_OUT_OF_MEMORY;
    }

    struct _fifoPrivate_t *handle = (struct _fifoPrivate_t *) malloc(sizeof(struct _fifoPrivate_t));
    (*fifoHandle)->private_data = handle;
    if (!handle) {
        mvLog(MVLOG_ERROR, "Memory allocation failed");
        return NC_OUT_OF_MEMORY;
    }

    handle->type = type;
    handle->consumer_cnt = 1;   //default consumers

    handle->state = NC_FIFO_CREATED;
    CHECK_MUTEX_SUCCESS(pthread_mutex_init(&handle->fifo_mutex, NULL));
    handle->consumed_by_graph = 0;
    handle->write_count = 0;
    handle->user_param_in = NULL;
    handle->user_param_out = NULL;
    handle->api_read_element = 0;
    handle->id = fifoIdCounter++;
    handle->num_elements = 0;
    memset(&handle->host_tensor_desc, 0, sizeof(struct ncTensorDescriptor_t));
    mv_strncpy(handle->name, NC_MAX_NAME_SIZE, name, NC_MAX_NAME_SIZE - 1);

    return NC_OK;
}

int pushUserParam(struct _fifoPrivate_t *fH, void *user_param, int isIn)
{
    struct _userParamPrivate_t *new_user_param =
        calloc(1, sizeof(struct _userParamPrivate_t));
    if (!new_user_param) {
        mvLog(MVLOG_ERROR, "Memory allocation failed");
        return NC_OUT_OF_MEMORY;
    }
    new_user_param->next = NULL;
    new_user_param->data = user_param;
    if (isIn) {
        new_user_param->next = fH->user_param_in;
        fH->user_param_in = new_user_param;
    } else {
        new_user_param->next = fH->user_param_out;
        fH->user_param_out = new_user_param;
    }
    return NC_OK;
}
int popUserParam(struct _fifoPrivate_t* fH, void** user_param, int isIn)
{
    struct _userParamPrivate_t* prev = NULL;
    struct _userParamPrivate_t* curr = NULL;
    if (isIn)
        curr = fH->user_param_in;
    else
        curr = fH->user_param_out;

    if (curr == NULL) {
        *user_param = NULL;
        mvLog(MVLOG_ERROR, "Trying to read user param from an empty queue!");
        return NC_ERROR;
    }

    while (curr->next != NULL)
    {
        prev = curr;
        curr = curr->next;
    }

    *user_param = curr->data;

    if (prev)
        prev->next = NULL;
    else {
        if (isIn)
            fH->user_param_in = NULL;
        else
            fH->user_param_out = NULL;
    }
    free(curr);
    curr = NULL;
    return NC_OK;
}

ncStatus_t ncFifoAllocate(struct ncFifoHandle_t * fifoHandle,
                          struct ncDeviceHandle_t * device,
                          struct ncTensorDescriptor_t * tensor_desc,
                          unsigned int numElem)
{
    mvLog(MVLOG_INFO, "Creating fifo");
    CHECK_HANDLE_CORRECT(fifoHandle);
    CHECK_HANDLE_CORRECT(device);

    if (!tensor_desc || !numElem) {
        mvLog(MVLOG_ERROR, "Some of the parameters are NULL");
        return NC_INVALID_PARAMETERS;
    }
    if (tensor_desc->n * tensor_desc->c * tensor_desc->w * tensor_desc->h == 0
        || !tensor_desc->totalSize) {
        mvLog(MVLOG_ERROR,
              "Tensor descriptor is invalid. Total size 0 or other element is zero");
        return NC_INVALID_PARAMETERS;
    }
    struct _fifoPrivate_t *handle = fifoHandle->private_data;
    if (handle->state == NC_FIFO_ALLOCATED) {
        mvLog(MVLOG_ERROR, "Fifo has already been allocated");
        return NC_UNAUTHORIZED;
    }
    if (handle->state != NC_FIFO_CREATED) {
        mvLog(MVLOG_ERROR, "Fifo handle is corrupt or has been destroyed");
        return NC_INVALID_HANDLE;
    }
    struct _devicePrivate_t *d = devices;
    GLOBAL_LOCK();
    while (d) {
        if (d == device->private_data)
            break;
        d = d->next;
    }
    if (!d) {
        GLOBAL_UNLOCK();
        mvLog(MVLOG_ERROR, "Device not found!\n");
        return NC_INVALID_PARAMETERS;
    }
    GLOBAL_UNLOCK();

    ncStatus_t rc = NC_OK;
    handle->graph_tensor_desc = *tensor_desc;
    handle->host_tensor_desc = *tensor_desc;
    handle->user_param_in = NULL;
    handle->user_param_out = NULL;
    handle->num_elements = numElem;
    handle->consumers_remaining = handle->consumer_cnt; //default consumers
    handle->dev = d;
    handle->next = NULL;

    handle->datasize = handle->host_tensor_desc.totalSize;

    if (d->fifos)
        handle->next = d->fifos;
    d->fifos = handle;

    bufferAllocateCommand_t cmd;
    cmd.type = GRAPH_BUFFER_ALLOCATE_CMD;
    struct tensorDescriptor_t privateDesc;
    privateDesc.c = tensor_desc->c;
    privateDesc.n = tensor_desc->n;
    privateDesc.h = tensor_desc->h;
    privateDesc.w = tensor_desc->w;
    // should be removiedd: #-17902
    privateDesc.totalSize = tensor_desc->totalSize;

    cmd.desc  = privateDesc;
    cmd.elemCnt = numElem;
    snprintf(cmd.name, MAX_STREAM_NAME_LENGTH, "FIFO%d", handle->id);
    cmd.id = handle->id;

    uint32_t writeSize;
    if (fifoWriteAccess(handle)) {
        writeSize = tensor_desc->totalSize * numElem;
        cmd.writeChannel = 1;
    } else {
        cmd.writeChannel = 0;
        writeSize = 8; // no write permission on this buffer, so we shouldn't bother allocating buffer on the device
    }
    if (fifoReadAccess(handle)) {
        cmd.readChannel = 1;
    } else {
        cmd.readChannel = 0;
    }
    streamId_t streamId = XLinkOpenStream(d->xlink->linkId, cmd.name, writeSize);

    char out_msg[NC_MAX_NAME_SIZE * 2];
    snprintf(out_msg, NC_MAX_NAME_SIZE * 2, "%s %s", "can't open stream: ", cmd.name);

    CHECK_STREAM_ID(streamId, {
            handle->state = NC_FIFO_FAILED;
            handle->dev->state = NC_DEVICE_FAILED;
        }, out_msg);

    handle->streamId = streamId;
    CHECK_MUTEX_SUCCESS(pthread_mutex_lock(&d->graph_stream_m));

    rc = trySendCommand(d->graph_monitor_stream_id, &cmd, sizeof(cmd));
    if(rc != 0){
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->graph_stream_m));
        mvLog(MVLOG_ERROR, "can't send command\n");
        return rc;
    }
    if (checkGraphMonitorResponse(d->graph_monitor_stream_id)) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->graph_stream_m));
        mvLog(MVLOG_ERROR, "myriad NACK\n");
        return NC_ERROR;
    }
    CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->graph_stream_m));

    handle->state = NC_FIFO_ALLOCATED;
    return NC_OK;

}

ncStatus_t ncFifoDestroy(struct ncFifoHandle_t ** fifoHandle)
{
    CHECK_HANDLE_CORRECT(fifoHandle);
    struct ncFifoHandle_t *fh = *fifoHandle;
    if (!fh) {
        mvLog(MVLOG_INFO, "handle is already destroyed");
        return NC_OK;
    }

    struct _fifoPrivate_t *handle = fh->private_data;

    if (handle->state == NC_FIFO_CREATED || handle->state == NC_FIFO_DEALLOCATED) {
        pthread_mutex_t * fifo_mutex = &fh->private_data->fifo_mutex;
#if !(defined(_WIN32) || defined(_WIN64))
        /**
         * There is no wrapper for pthread_mutex_trylock on windows at the moment.
         */
        int error = pthread_mutex_trylock(fifo_mutex);
        if (error && error != EBUSY) {
            /**
             * Calling pthread_mutex_unlock with not locked mutex is undefined behavior.
             * There is no standard C-API functions for checking whether mutex is locked or not as well as state entry.
             * After pthread_mutex_trylock mutex can be safely unlocked since it is already locked.
             * EBUSY error code stands for already locked mutex that is not an error in this case.
             */
             mvLog(MVLOG_ERROR, "pthread_mutex_trylock(fifo_mutex) failed with error: %d", error);
        }
#endif

        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(fifo_mutex));
        CHECK_MUTEX_SUCCESS(pthread_mutex_destroy(fifo_mutex));

        free(fh->private_data);
        fh->private_data = NULL;

        free(fh);
        *fifoHandle = NULL;

        return NC_OK;
    }
    if (!findFifo(handle)) {
        mvLog(MVLOG_ERROR,
              "fifo handle seems to be corrupt or has been destroyed");
        return NC_INVALID_HANDLE;
    }
    //clean up fifo
    /*if (fifoReadAccess(handle)) {
        int fillLevel;
        int rc = XLinkGetFillLevel(handle->streamId, 0, &fillLevel);
        if (rc == X_LINK_SUCCESS) {
            while (fillLevel && rc == X_LINK_SUCCESS) {
                rc = XLinkReleaseData(handle->streamId);
                fillLevel--;
            }
        }
    }*/
    //First write to the fifo to stop it's thread
    if (fifoWriteAccess(handle)) {
        int msg = 0xdead;
        if (XLinkWriteData(handle->streamId, (uint8_t *) & msg, sizeof(msg)) !=
            0) {
            mvLog(MVLOG_ERROR, "Failed to write to fifo before deleting it!");
            return NC_ERROR;
        }
    }

    ncStatus_t rc = NC_OK;
    graphCommonCommand_t cmd;
    cmd.type = GRAPH_BUFFER_DEALLOCATE_CMD;
    cmd.id = handle->id;

    struct _devicePrivate_t *d = handle->dev;
    CHECK_MUTEX_SUCCESS(pthread_mutex_lock(&d->graph_stream_m));
    rc = trySendCommand(d->graph_monitor_stream_id, &cmd, sizeof(cmd));
    if(rc != 0){
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->graph_stream_m));
        mvLog(MVLOG_WARN, "can't send command\n");
        return rc;
    }
    if (checkGraphMonitorResponse(d->graph_monitor_stream_id)) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->graph_stream_m));
        mvLog(MVLOG_WARN, "myriad NACK\n");
        return NC_ERROR;
    }
    CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->graph_stream_m));

    CHECK_MUTEX_SUCCESS(pthread_mutex_lock(&d->dev_data_m));
    if (deallocateFifo(handle)) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_data_m));
        return NC_INVALID_PARAMETERS;
    }
    CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&d->dev_data_m));

    free(fh->private_data);
    fh->private_data = NULL;
    free(fh);
    *fifoHandle = NULL;
    return NC_OK;

}

ncStatus_t ncFifoWriteElem(struct ncFifoHandle_t * fifoHandle,
                           const void *inputTensor,
                           unsigned int * inputTensorLength,
                           void *userParam)
{
    CHECK_HANDLE_CORRECT(fifoHandle);

    if (inputTensorLength == NULL || *inputTensorLength <= 0) {
        mvLog(MVLOG_ERROR, "inputTensorSize is null or invalid value");
        return NC_INVALID_PARAMETERS;
    }
    struct _fifoPrivate_t *handle = fifoHandle->private_data;
    if (!findFifo(handle)) {
        if (!handle) {
            mvLog(MVLOG_ERROR,
                  "fifo handle seems to be corrupt or has been destroyed");
            return NC_INVALID_HANDLE;
        }
        if (handle->state == NC_FIFO_CREATED) {
            mvLog(MVLOG_ERROR, "FIFO is not yet allocated");
            return NC_NOT_ALLOCATED;
        }
        if (handle->state != NC_FIFO_ALLOCATED) {
            mvLog(MVLOG_ERROR,
                  "FIFO is not yet allocated or have been destroyed.");
            return NC_UNAUTHORIZED;
        }
    }

    CHECK_HANDLE_CORRECT_RC(inputTensor, NC_INVALID_PARAMETERS);

    if (!fifoWriteAccess(handle)) {
        mvLog(MVLOG_ERROR, "No write access to fifo");
        return NC_UNAUTHORIZED;
    }
    if (*inputTensorLength != handle->datasize) {
            mvLog(MVLOG_ERROR,
                  "input tensor length (%d) doesnt match expected value (%d)",
                  *inputTensorLength, handle->datasize);
            *inputTensorLength = handle->datasize;
            return NC_INVALID_DATA_LENGTH;
    }
    int rc = XLinkWriteData(handle->streamId, inputTensor, *inputTensorLength);
    if (rc != 0)
        return NC_ERROR;

    CHECK_MUTEX_SUCCESS_RC(pthread_mutex_lock(&handle->fifo_mutex), NC_ERROR);
    rc = pushUserParam(handle, userParam, 1);
    if (rc != NC_OK) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&handle->fifo_mutex));
        return rc;
    }
    handle->write_count++;
    CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&handle->fifo_mutex));

    mvLog(MVLOG_DEBUG, "write count %d num_elements %d userparam %p\n",
          handle->write_count - 1, handle->num_elements, userParam);
    return NC_OK;

}

ncStatus_t ncFifoReadElem(struct ncFifoHandle_t * fifoHandle, void *outputData,
                          unsigned int *outputDataLen, void **userParam)
{
    if (!fifoHandle) {
        mvLog(MVLOG_ERROR, "fifo handle is NULL");
        return NC_INVALID_HANDLE;
    }
    if (!outputDataLen || (*outputDataLen != 0 && !outputData)) {
        mvLog(MVLOG_ERROR, "Some of the parameters are NULL");
        return NC_INVALID_PARAMETERS;
    }

    struct _fifoPrivate_t *handle = fifoHandle->private_data;
    if (!findFifo(handle)) {
        if (!handle) {
            mvLog(MVLOG_ERROR,
                  "fifo handle seems to be corrupt or has been destroyed");
            return NC_INVALID_HANDLE;
        }
        if (handle->state == NC_FIFO_CREATED) {
            mvLog(MVLOG_ERROR, "FIFO is not yet allocated");
            return NC_NOT_ALLOCATED;
        }
    }

    if (handle->state != NC_FIFO_ALLOCATED) {
        mvLog(MVLOG_ERROR, "FIFO is not yet allocated or have been destroyed.");
        return NC_UNAUTHORIZED;
    }

    if (*outputDataLen < handle->datasize) {
        mvLog(MVLOG_ERROR,
              "This datasize in tensorDesc (%d) is smaller than required (%d)!",
              *outputDataLen, handle->datasize);
        *outputDataLen = handle->datasize;
        return NC_INVALID_DATA_LENGTH;
    }

    if (!fifoReadAccess(handle)) {
        mvLog(MVLOG_ERROR, "FIFO has no read access");
        return NC_UNAUTHORIZED;
    }
    if (handle->api_read_element != 0) {
        mvLog(MVLOG_ERROR, "API already read this element");
        return NC_UNAUTHORIZED;
    }
    streamPacketDesc_t *packet = 0;
    if (!XLinkReadData(handle->streamId, &packet) && packet) {
        mvnc_memcpy(outputData, *outputDataLen, packet->data, packet->length);
        XLinkReleaseData(handle->streamId);
    } else {
        mvLog(MVLOG_ERROR, "Packet reading is failed.");
        return NC_ERROR;
    }

    //As user should see an API read to be the same as Graph read, we need to write the element in 2 queues.
    //if we read it here, we will need to remove the element on the device side
    //to avoid sending a message just for this purpose, we can send it at the next trigger which touches this FIFO.
    CHECK_MUTEX_SUCCESS_RC(pthread_mutex_lock(&handle->fifo_mutex), NC_ERROR);
    handle->api_read_element = 1;

    handle->consumers_remaining--;
    if (handle->consumers_remaining == 0) {
        handle->api_read_element = 0;
        handle->consumers_remaining = handle->consumer_cnt;
        //no other action required when the element is consumed
    }
    popUserParam(handle, userParam, 0);
    CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&handle->fifo_mutex));
    *outputDataLen = handle->datasize;
    mvLog(MVLOG_DEBUG, "num_elements %d userparam %p output length %d\n",
          handle->num_elements, userParam, handle->datasize);
    return NC_OK;
}

static ncStatus_t tensorCompatibility(struct ncTensorDescriptor_t *tens1,
                                      struct ncTensorDescriptor_t *tens2)
{
    if (tens1->totalSize != tens2->totalSize ||
        tens1->n != tens2->n || tens1->c != tens2->c ||
        tens1->h != tens2->h || tens1->w != tens2->w)
        return NC_ERROR;
    return NC_OK;
}

ncStatus_t ncGraphQueueInference(struct ncGraphHandle_t * graphHandle,
                                 struct ncFifoHandle_t ** fifoIn,
                                 unsigned int inFifoCount,
                                 struct ncFifoHandle_t ** fifoOut,
                                 unsigned int outFifoCount)
{
    mvLog(MVLOG_DEBUG, "Trigger start");
    CHECK_HANDLE_CORRECT(graphHandle);
    CHECK_HANDLE_CORRECT(fifoIn);
    CHECK_HANDLE_CORRECT(fifoOut);

    if (!fifoIn[0] || !fifoOut[0]) {
        mvLog(MVLOG_ERROR, "Fifos data are NULL");
        return NC_INVALID_HANDLE;
    }
    if (!inFifoCount || !outFifoCount)
        return NC_INVALID_PARAMETERS;

    struct _graphPrivate_t *g = graphHandle->private_data;

    if(g) {
        CHECK_MUTEX_SUCCESS_RC(pthread_mutex_lock(&g->dev->graph_stream_m), NC_ERROR);
    } else {
        return NC_NOT_ALLOCATED;
    }

    if (!g || g->state != NC_GRAPH_ALLOCATED) {
        mvLog(MVLOG_ERROR, "Graph hasn't been allocated");
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
        return NC_NOT_ALLOCATED;
    }

    if (g->input_count != inFifoCount || g->output_count != outFifoCount) {
        mvLog(MVLOG_ERROR,
              "number of input or output fifos is not compatible with graph");
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
        return NC_INVALID_PARAMETERS;
    }

    if (inFifoCount != 1 || outFifoCount != 1) {
        mvLog(MVLOG_ERROR,
              "Currently multiple inputs and outputs are not supported");
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
        return NC_UNSUPPORTED_FEATURE;
    }
    struct _fifoPrivate_t *fi = fifoIn[0]->private_data;
    struct _fifoPrivate_t *fo = fifoOut[0]->private_data;
    ncStatus_t rc;
    if (fi->state != NC_FIFO_ALLOCATED || fo->state != NC_FIFO_ALLOCATED) {
        mvLog(MVLOG_ERROR, "ffos hasn't been allocated");
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
        return NC_NOT_ALLOCATED;
    }
    //WO fifos have no graph access
    if (fo->type == NC_FIFO_HOST_WO) {
        //graphs have no access to one of the fifos
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
        return NC_INVALID_PARAMETERS;
    }
    if (tensorCompatibility(&fi->graph_tensor_desc, &g->input_tensor_desc) != NC_OK ||
        tensorCompatibility(&fo->graph_tensor_desc,
                            &g->output_tensor_desc) != NC_OK) {
        mvLog(MVLOG_WARN,
              "Input/Output tensor shape is not compatible with graph");
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
        return NC_INVALID_PARAMETERS;
    }

    graphCMDCommand_t cmd;
    cmd.type = GRAPH_TRIGGER_CMD;
    cmd.id = g->id;
    cmd.buffId1 = fi->id;
    cmd.buffId2 = fo->id;

    void* user_param;
    CHECK_MUTEX_SUCCESS_RC(pthread_mutex_lock(&fi->fifo_mutex), NC_ERROR);
    fi->consumers_remaining--;

    if (fi->consumers_remaining == 0) {
        if (!fi->api_read_element && fifoReadAccess(fi)) {//the element was entirely consumed by graphs. This means we need to free it up from XLink
            streamPacketDesc_t* packet = 0;
            XLinkError_t rc = XLinkReadData(fi->streamId, &packet);
            if (rc) {
                mvLog(MVLOG_ERROR, "Can't read packet, rc: %s", XLinkErrorToStr(rc));
                CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&fi->fifo_mutex));
                fi->dev->state = NC_DEVICE_FAILED;
                CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
                return parseXLinkError(rc);
            }
            rc = XLinkReleaseData(fi->streamId);
            if (rc) {
                mvLog(MVLOG_ERROR,"Failed to release data, rc: %s", XLinkErrorToStr(rc));
                CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&fi->fifo_mutex));
                fi->dev->state = NC_DEVICE_FAILED;
                CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
                return parseXLinkError(rc);
            }
        }
        fi->consumers_remaining = fi->consumer_cnt;
        fi->api_read_element = 0;
    }
    popUserParam(fi, &user_param, 1);
    if (fi->write_count <= fi->consumed_by_graph) {
        mvLog(MVLOG_WARN, "No point on triggering graph. There are no more elements in the input FIFO");
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&fi->fifo_mutex));
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
        return NC_UNAUTHORIZED;
    }
    fi->consumed_by_graph++;
    CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&fi->fifo_mutex));

    CHECK_MUTEX_SUCCESS_RC(pthread_mutex_lock(&fo->fifo_mutex), NC_ERROR);
    rc = pushUserParam(fo, user_param , 0);
    if(rc != NC_OK) {
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&fo->fifo_mutex));
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
        return rc;
    }
    fo->write_count++;
    CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&fo->fifo_mutex));

    rc = trySendCommand(g->dev->graph_monitor_stream_id, &cmd, sizeof(cmd));
    if(rc != 0){
        mvLog(MVLOG_ERROR, "Can't send trigger request");
        g->dev->state = NC_DEVICE_FAILED;
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
        return rc;
    }
    if(checkGraphMonitorResponse(g->dev->graph_monitor_stream_id)) {
        mvLog(MVLOG_ERROR, "Can't get trigger response");
        g->dev->state = NC_DEVICE_FAILED;
        CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
        return NC_ERROR;
    }
    CHECK_MUTEX_SUCCESS(pthread_mutex_unlock(&g->dev->graph_stream_m));
    g->started = 1;
    mvLog(MVLOG_DEBUG, "Trigger end");
    return NC_OK;
}

ncStatus_t ncGraphQueueInferenceWithFifoElem(struct ncGraphHandle_t *
                                             graphHandle,
                                             struct ncFifoHandle_t * fifoIn,
                                             struct ncFifoHandle_t * fifoOut,
                                             const void *inputTensor,
                                             unsigned int * inputTensorLength,
                                             void *userParam)
{
    ncStatus_t rc = ncFifoWriteElem(fifoIn, inputTensor, inputTensorLength,
                                    userParam);
    if (rc != NC_OK)
        return rc;

    return ncGraphQueueInference(graphHandle, &fifoIn, 1, &fifoOut, 1);
}
