AWS Batchで速く/安くやるデータセットの前処理

FROM alpine:3.6

ENV OPENCV_VERSION 3.2.0
ENV PYTHON_VERSION 3.6.3

RUN apk add --update --no-cache \
    build-base \
    openblas \
    openblas-dev \
    unzip \
    curl \
    cmake \
    libjpeg \
    libjpeg-turbo-dev \
    libpng-dev \
    jasper-dev \
    tiff-dev \
    libwebp-dev \
    clang-dev \
    openssl \
    ncurses \
    libstdc++ \
    linux-headers && \
    mkdir -p /opt && \
    apk add --no-cache --virtual .build-python \
      openssl-dev \
      ncurses-dev \
      xz && \
    curl -LO https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz && \
    unxz Python-${PYTHON_VERSION}.tar.xz && \
    tar xf Python-${PYTHON_VERSION}.tar -C /opt && \
    cd /opt/Python-${PYTHON_VERSION} && \
    ./configure --enable-shared && make -j > /dev/null && make install > /dev/null && \
    cd / && \
    rm -rf /opt/Python-${PYTHON_VERSION} Python-${PYTHON_VERSION}.tar && \
    pip3 install numpy==1.13.3 && \
    apk del --virtual .build-python && \
    curl -LO https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
    unzip ${OPENCV_VERSION}.zip -d /opt && \
    rm -rf ${OPENCV_VERSION}.zip && \
    mkdir -p /opt/opencv-${OPENCV_VERSION}/build && \
    cd /opt/opencv-${OPENCV_VERSION}/build && \
    cmake \
      -D CMAKE_BUILD_TYPE=RELEASE \
      -D CMAKE_INSTALL_PREFIX=/usr/local \
      -D WITH_FFMPEG=NO \
      -D WITH_IPP=NO \
      -D WITH_OPENEXR=NO \
      -D WITH_TBB=YES \
      -D BUILD_TESTS=NO \
      -D BUILD_EXAMPLES=NO \
      -D BUILD_ANDROID_EXAMPLES=NO \
      -D INSTALL_PYTHON_EXAMPLES=NO \
      -D BUILD_DOCS=NO \
      -D BUILD_opencv_python2=NO \
      -D BUILD_opencv_python3=ON \
      -D PYTHON3_EXECUTABLE=/usr/local/bin/python3 \
      -D PYTHON3_INCLUDE_DIR=/usr/local/include/python3.6m/ \
      -D PYTHON3_LIBRARY=/usr/local/lib/libpython3.so \
      -D PYTHON_LIBRARY=/usr/local/lib/libpython3.so \
      -D PYTHON3_PACKAGES_PATH=/usr/local/lib/python3.6/site-packages/ \
      -D PYTHON3_NUMPY_INCLUDE_DIRS=/usr/local/lib/python3.6/site-packages/numpy/core/include/ \
      .. && \
    make VERBOSE=1 && \
    make -j2 && \
    make install && \
    cd / && \
    rm -rf /opt && \
    apk del gcc build-base openblas-dev clang-dev linux-headers cmake unzip

  ComputeEnvironment:
    Type: AWS::Batch::ComputeEnvironment
    Properties:
      Type: MANAGED
      ServiceRole: !Sub "arn:aws:iam::${AWS::AccountId}:role/AWSBatchServiceRole"
      ComputeEnvironmentName: C4OnDemand
      ComputeResources:
        MaxvCpus: 128
        SecurityGroupIds:
          - !Ref InstanceSecurityGroup
        Type: EC2
        Subnets:
          - !Ref PrivateSubnet
        MinvCpus: 0
        ImageId: ami-a1b2c3d4
        InstanceRole: ecsInstanceRole
        InstanceTypes:
          - c4.large
          - c4.xlarge
          - c4.2xlarge
          - c4.4xlarge
          - c4.8xlarge
        Ec2KeyPair: batch-compute
        Tags: {"Name": "Batch Instance - C4OnDemand"}
        DesiredvCpus: 48
      State: ENABLED

  MyRepository: 
    Type: "AWS::ECR::Repository"
    Properties: 
      RepositoryName: "image-convert"
      RepositoryPolicyText:
        Version: "2012-10-17"
        Statement: 
          - 
            Sid: AllowPushPull
            Effect: Allow
            Principal:
              AWS: !Sub "arn:aws:iam::${AWS::AccountId}:user/${User}"
            Action: 
              - "ecr:GetDownloadUrlForLayer"
              - "ecr:BatchGetImage"
              - "ecr:BatchCheckLayerAvailability"
              - "ecr:PutImage"
              - "ecr:InitiateLayerUpload"
              - "ecr:UploadLayerPart"
              - "ecr:CompleteLayerUpload"
          - 
            Sid: AllowPull
            Effect: Allow
            Principal:
              AWS: !GetAtt JobRole.Arn
            Action: 
              - "ecr:GetDownloadUrlForLayer"
              - "ecr:BatchGetImage"
              - "ecr:BatchCheckLayerAvailability"

  JobDefinitionForResize:
    Type: 'AWS::Batch::JobDefinition'
    Properties:
      Type: container
      JobDefinitionName: !Sub
          - ${Service}-resize-image
          - { Service: !Ref ServiceName}
      Parameters:
        Bucket: !Ref Bucket
        ExcludeFiles: exclude.txt
        Size: 128
      ContainerProperties:
        Command:
          - python3
          - -m
          - resize_fix_size
          - -b
          - "Ref::Bucket"
          - -d
          - resized
          - -e
          - "Ref::ExcludeFiles"
          - -s
          - "Ref::Size"
          - --crop
          - "Ref::Prefix"
        Memory: 1000
        JobRoleArn: !Ref JobRole
        Vcpus: 1
        Image: !Sub
          - "${AWS::AccountId}.dkr.ecr.ap-northeast-1.amazonaws.com/${Repository}/image-converter"
          - {"Repository": !Ref MyRepository}
      RetryStrategy:
        Attempts: 1

  JobQueue:
    Type: AWS::Batch::JobQueue
    Properties:
      ComputeEnvironmentOrder:
        - Order: 1
          ComputeEnvironment: !Ref ComputeEnvironment
      State: ENABLED
      Priority: 1
      JobQueueName: HighPriority

# submit-all-jobs.sh
#!/bin/bash
JOB_QUEUE=xxxxxx

RESIZE_JOB_ARN=$(aws batch describe-job-definitions --job-definition-name image-converter-resize-image --status ACTIVE | jq -r '.jobDefinitions | max_by(.revision).jobDefinitionArn')
EDGE_JOB_ARN=$(aws batch describe-job-definitions --job-definition-name image-converter-extract-edge --status ACTIVE | jq -r '.jobDefinitions | max_by(.revision).jobDefinitionArn')

MAKE_SEQ=$(cat <<EOF
for i in range(0, 0xff):
    print("{:0>2x}".format(i + 1))
EOF
        )

SEQ=$(python -c "$MAKE_SEQ")
echo "$SEQ" | xargs -P 8 -I {} -n 1 \
                    ./submit-jobs.sh $JOB_QUEUE $RESIZE_JOB_ARN $EDGE_JOB_ARN {}

# submit-jobs.sh
#!/bin/bash
JOB_QUEUE=$1
RESIZE_JOB_ARN=$2
EDGE_JOB_ARN=$3
SEQ=$4

resize_job=$(aws batch submit-job --job-name "resize-$SEQ-$now" --job-queue $JOB_QUEUE \
    --job-definition $RESIZE_JOB_ARN \
    --parameters Prefix=full/$SEQ
          )

resize_job_id=$(jq '.jobId' "$resize_job")

aws batch submit-job --job-name "edge-$SEQ-$now" --job-queue $JOB_QUEUE \
    --job-definition $RESIZE_JOB_ARN \
    --parameters Prefix=resized/$SEQ \
    --depends-on jobId=$resize_job_id