マイコンでディープラーニングした話 on ESP32

def mnist_lenet_prediction(image, test=False, aug=None):
    """
    Construct LeNet for MNIST.
    """
    image /= 255.0
    image = augmentation(image, test, aug)
    c1 = PF.convolution(image, 16, (5, 5), name='conv1')
    c1 = F.relu(F.max_pooling(c1, (2, 2)), inplace=True)
    c2 = PF.convolution(c1, 16, (5, 5), name='conv2')
    c2 = F.relu(F.max_pooling(c2, (2, 2)), inplace=True)
    c3 = F.relu(PF.affine(c2, 50, name='fc3'), inplace=True)
    c4 = PF.affine(c3, 10, name='fc4')
    return c4

git clone https://github.com/sony/nnabla-examples.git
cd nnabla-examples/mnist-collection
python classification.py -c cudnn -n lenet -o output

mkdir ./output_csrc
nnabla_cli convert -O CSRC -b 1 ./output/lenet_result.nnp ./output_csrc

.
└── output_csrc
    ├── GNUmakefile                  テスト用Makefile
    ├── Validation_example.c         テスト用コード
    ├── Validation_inference.c       モデル定義類
    ├── Validation_inference.h       ↑のヘッダ
    ├── Validation_parameters.c      重みパラメータ類
    └── Validation_parameters.h      ↑のヘッダ

cp -r ./output_csrc/* ~/esp/esp32-camera-dnn/components/dnn/

cd ~/esp/esp32-camera-dnn/components/dnn/
git submodule add https://github.com/sony/nnabla-c-runtime.git

    .pixel_format = PIXFORMAT_GRAYSCALE, //YUV422,GRAYSCALE,RGB565,JPEG
    .frame_size = FRAMESIZE_QQVGA, //QQVGA-UXGA Do not use sizes above QVGA when not JPEG

    .jpeg_quality = 20, //0-63 lower number means higher quality
    .fb_count = 1 //if more than one, i2s runs in continuous mode. Use only with JPEG

esp_err_t jpg_stream_httpd_handler(httpd_req_t *req) {
    // タスク初期化
    // ...

    float *nn_input_buffer = nnablart_validation_input_buffer(_context, 0);

    while (true) {
        // カメラ画像取得
        camera_fb_t *fb = esp_camera_fb_get();

        // ...

        // リサイズ
        stbir_resize_uint8(fb->buf, 160, 120, 0, resized_img, 28, 28, 0, 1);
        esp_camera_fb_return(fb);

        // 白黒反転, 閾値処理
        for (int i = 0; i < NNABLART_VALIDATION_INPUT0_SIZE; i++) {
            uint8_t p = ~(resized_img[i]);

            if (p < 180) {
                p = 0;
            }

            nn_input_buffer[i] = p;
        }

        // 推論実行
        nnablart_validation_inference(_context);

        // 推論結果の取得
        float *pred = nnablart_validation_output_buffer(_context, 0);
        int top_class = 0;
        float top_probability = 0.0f;
        for (int class = 0; class < NNABLART_VALIDATION_OUTPUT0_SIZE; class++) {
            if (top_probability < pred[class]) {
                top_probability = pred[class];
                top_class = class;
            }
        }

        // ...

        // 推論結果, 処理時間送信
        ESP_LOGI(TAG, "Result %d   Frame-time %ums (Inferrence-time %ums)",
            top_class, (uint32_t)frame_time, (uint32_t)infer_time);
    }

    // タスク終了
    nnablart_validation_free_context(_context);

    // ...
}

make monitor PRINT_FILTER="camera"

def mnist_lenet_light_prediction(image, test=False, aug=None):
    """
    Construct LeNet for MNIST.
    """
    image /= 255.0
    image = augmentation(image, test, aug)
    c1 = PF.convolution(image, 4, (3, 3), name='conv1')
    c1 = F.relu(F.max_pooling(c1, (2, 2)), inplace=True)
    c2 = PF.convolution(c1, 16, (1, 1), name='conv2')
    c2 = PF.depthwise_convolution(c2, (3, 3), name='dw_conv2')
    c2 = F.relu(F.max_pooling(c2, (2, 2)), inplace=True)
    c3 = F.relu(PF.affine(c2, 30, name='fc3'), inplace=True)
    c4 = PF.affine(c3, 10, name='fc4')
    return c4