custom3_training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 続・detectron2 for まちカドまぞく ～カスタム訓練編～\n",
    "\n",
    "<img src=\"https://user-images.githubusercontent.com/33882378/79110283-0f737980-7db5-11ea-89b0-3b6b19ae1716.jpg\">\n",
    "\n",
    "detectron2 の訓練をカスタマイズする方法\n",
    "\n",
    "ここでは、「カスタムトレーナー」を作りますが、今までの工程では一番楽でした。。\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import json\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "##  VoTT Export からの読み込み"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# VoTT のエクスポートファイルや、画像が格納されているディレクトリ\n",
    "BASE_DIRECTORY = './vott-json-export/'\n",
    "# VoTT のエクスポートファイル名\n",
    "EXPORT_FILENAME = 'Machikado-export.json'\n",
    "# 訓練データに使用する割合\n",
    "TRAIN_RATIO = 0.8\n",
    "# 乱数シード\n",
    "RANDOM_STATE = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "警告: name: 59.jpg - 画像サイズが不一致であるためスキップ image_size:(268, 201), ./vott-json-export/Machikado-export.json: (600, 600)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Metadata(name='test', thing_classes=['Shamiko', 'Gosenzo', 'Lilith', 'Momo', 'Mikan', 'Mob'])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from detectron2.data import DatasetCatalog, MetadataCatalog\n",
    "from machikado_util.Machikado_vott import get_cat_names, get_machikado_dicts\n",
    "\n",
    "# vott エクスポートファイルの読み込み\n",
    "CAT_NAME2ID, CAT_ID2NAME = get_cat_names(os.path.join(BASE_DIRECTORY, EXPORT_FILENAME))\n",
    "dataset_dicts = get_machikado_dicts(os.path.join(BASE_DIRECTORY, EXPORT_FILENAME), BASE_DIRECTORY, CAT_NAME2ID)\n",
    "\n",
    "# 訓練用、テスト用に分ける\n",
    "random.seed(RANDOM_STATE)\n",
    "random.shuffle(dataset_dicts)\n",
    "\n",
    "split_idx = int(len(dataset_dicts) * TRAIN_RATIO) + 1\n",
    "\n",
    "# 登録\n",
    "DatasetCatalog.clear()\n",
    "DatasetCatalog.register('train', lambda : dataset_dicts[:split_idx])\n",
    "DatasetCatalog.register('test', lambda : dataset_dicts[split_idx:])\n",
    "\n",
    "MetadataCatalog.get('train').set(thing_classes=list(CAT_NAME2ID.keys()))\n",
    "MetadataCatalog.get('test').set(thing_classes=list(CAT_NAME2ID.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from detectron2.data import DatasetCatalog\n",
    "\n",
    "# 訓練用、テスト用に分ける\n",
    "random.seed(RANDOM_STATE)\n",
    "random.shuffle(dataset_dicts)\n",
    "split_idx = int(len(dataset_dicts) * TRAIN_RATIO) + 1\n",
    "\n",
    "# 登録\n",
    "DatasetCatalog.clear()\n",
    "DatasetCatalog.register('train', lambda : dataset_dicts[:split_idx])\n",
    "DatasetCatalog.register('test', lambda : dataset_dicts[split_idx:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 学習\n",
    "\n",
    "* 前回より訓練回数を盛っています。\n",
    "* 入力サイズを小さくしたので、バッチ処理数を増やせます。 (1 -> 6 へ変更)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from detectron2.config import get_cfg\n",
    "from machikado_util.custom_config import append_custom_cfg\n",
    "\n",
    "cfg = get_cfg()\n",
    "append_custom_cfg(cfg)\n",
    "\n",
    "cfg.OUTPUT_DIR = './output'\n",
    "cfg.CUDA = 'cuda:0'\n",
    "\n",
    "cfg.merge_from_file('../configs/COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x.yaml')\n",
    "cfg.MODEL.WEIGHTS = './coco_models/model_final_2d9806.pkl'\n",
    "cfg.SOLVER.IMS_PER_BATCH = 6 # GTX2070 ではこれが限界\n",
    "\n",
    "cfg.DATASETS.TRAIN = ('train',)\n",
    "cfg.DATASETS.TEST = ()   # no metrics implemented for this dataset\n",
    "cfg.DATALOADER.NUM_WORKERS = 4\n",
    "cfg.SOLVER.BASE_LR = 0.00025\n",
    "cfg.SOLVER.MAX_ITER = 2500\n",
    "cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128   # faster, and good enough for this toy dataset\n",
    "cfg.MODEL.ROI_HEADS.NUM_CLASSES = len(CAT_ID2NAME) \n",
    "\n",
    "cfg.INPUT.MIN_SIZE_TRAIN = 400\n",
    "cfg.INPUT.MAX_SIZE_TRAIN = 640\n",
    "cfg.INPUT.MIN_SIZE_TEST = 400\n",
    "cfg.INPUT.MAX_SIZE_TEST = 640\n",
    "\n",
    "cfg.INPUT.CROP.ENABLED = True\n",
    "cfg.INPUT.CROP.SIZE = [0.8, 0.8]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### カスタムトレーナー\n",
    "\n",
    "* DefaultTrainer は detectron2/engine/defaults.py にあります。\n",
    "* DefaultTrainer を全部書き換える必要は無く、今回の場合であればデーターマッパーを渡す部分を変更するだけで良いです。（オーバーライドすれば良いです。）\n",
    "* あとは、同じです。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from machikado_util.MachikadoDatasetMapper import MachikadoDatasetMapper\n",
    "from detectron2.engine import DefaultTrainer\n",
    "\n",
    "class MachikadoTrainer(DefaultTrainer):\n",
    "    def __init__(self, cfg):\n",
    "        super().__init__(cfg)\n",
    "    \n",
    "    # ここをオーバーライドして、作った自分のデータマッパーを返すように変更する。\n",
    "    @classmethod\n",
    "    def build_train_loader(cls, cfg):\n",
    "        return build_detection_train_loader(cfg, MachikadoDatasetMapper(cfg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[04/19 05:12:11 d2.data.detection_utils]: \u001b[0mTransformGens used in training: [ResizeShortestEdge(short_edge_length=(400, 400), max_size=640, sample_style='choice'), RandomFlip()]\n",
      "\u001b[32m[04/19 05:12:11 d2.data.build]: \u001b[0mRemoved 0 images with no usable annotations. 92 images left.\n",
      "\u001b[32m[04/19 05:12:11 d2.data.build]: \u001b[0mDistribution of instances among all 6 categories:\n",
      "\u001b[36m|  category  | #instances   |  category  | #instances   |  category  | #instances   |\n",
      "|:----------:|:-------------|:----------:|:-------------|:----------:|:-------------|\n",
      "|  Shamiko   | 74           |  Gosenzo   | 28           |   Lilith   | 11           |\n",
      "|    Momo    | 42           |   Mikan    | 15           |    Mob     | 19           |\n",
      "|            |              |            |              |            |              |\n",
      "|   total    | 189          |            |              |            |              |\u001b[0m\n",
      "\u001b[32m[04/19 05:12:11 d2.data.common]: \u001b[0mSerializing 92 elements to byte tensors and concatenating them all ...\n",
      "\u001b[32m[04/19 05:12:11 d2.data.common]: \u001b[0mSerialized dataset takes 0.24 MiB\n",
      "\u001b[32m[04/19 05:12:11 d2.data.build]: \u001b[0mUsing training sampler TrainingSampler\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "'roi_heads.box_predictor.cls_score.weight' has shape (81, 1024) in the checkpoint but (7, 1024) in the model! Skipped.\n",
      "'roi_heads.box_predictor.cls_score.bias' has shape (81,) in the checkpoint but (7,) in the model! Skipped.\n",
      "'roi_heads.box_predictor.bbox_pred.weight' has shape (320, 1024) in the checkpoint but (24, 1024) in the model! Skipped.\n",
      "'roi_heads.box_predictor.bbox_pred.bias' has shape (320,) in the checkpoint but (24,) in the model! Skipped.\n",
      "'roi_heads.mask_head.predictor.weight' has shape (80, 256, 1, 1) in the checkpoint but (6, 256, 1, 1) in the model! Skipped.\n",
      "'roi_heads.mask_head.predictor.bias' has shape (80,) in the checkpoint but (6,) in the model! Skipped.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[04/19 05:12:13 d2.engine.train_loop]: \u001b[0mStarting training from iteration 0\n",
      "\u001b[32m[04/19 05:12:32 d2.utils.events]: \u001b[0m eta: 0:38:23  iter: 19  total_loss: 3.748  loss_cls: 2.147  loss_box_reg: 0.861  loss_mask: 0.694  loss_rpn_cls: 0.036  loss_rpn_loc: 0.009  time: 0.9274  data_time: 0.0319  lr: 0.000005  max_mem: 5446M\n",
      "\u001b[32m[04/19 05:12:51 d2.utils.events]: \u001b[0m eta: 0:38:04  iter: 39  total_loss: 3.567  loss_cls: 1.983  loss_box_reg: 0.834  loss_mask: 0.691  loss_rpn_cls: 0.037  loss_rpn_loc: 0.010  time: 0.9308  data_time: 0.0096  lr: 0.000010  max_mem: 5675M\n",
      "\u001b[32m[04/19 05:13:09 d2.utils.events]: \u001b[0m eta: 0:37:50  iter: 59  total_loss: 3.272  loss_cls: 1.663  loss_box_reg: 0.897  loss_mask: 0.686  loss_rpn_cls: 0.030  loss_rpn_loc: 0.010  time: 0.9329  data_time: 0.0096  lr: 0.000015  max_mem: 5697M\n",
      "\u001b[32m[04/19 05:13:28 d2.utils.events]: \u001b[0m eta: 0:37:35  iter: 79  total_loss: 2.865  loss_cls: 1.302  loss_box_reg: 0.876  loss_mask: 0.681  loss_rpn_cls: 0.031  loss_rpn_loc: 0.010  time: 0.9331  data_time: 0.0107  lr: 0.000020  max_mem: 5697M\n",
      "\u001b[32m[04/19 05:13:47 d2.utils.events]: \u001b[0m eta: 0:37:21  iter: 99  total_loss: 2.456  loss_cls: 0.921  loss_box_reg: 0.827  loss_mask: 0.666  loss_rpn_cls: 0.023  loss_rpn_loc: 0.007  time: 0.9343  data_time: 0.0095  lr: 0.000025  max_mem: 5706M\n",
      "\u001b[32m[04/19 05:14:06 d2.utils.events]: \u001b[0m eta: 0:37:05  iter: 119  total_loss: 2.375  loss_cls: 0.782  loss_box_reg: 0.867  loss_mask: 0.656  loss_rpn_cls: 0.037  loss_rpn_loc: 0.011  time: 0.9352  data_time: 0.0092  lr: 0.000030  max_mem: 5730M\n",
      "\u001b[32m[04/19 05:14:25 d2.utils.events]: \u001b[0m eta: 0:36:48  iter: 139  total_loss: 2.216  loss_cls: 0.671  loss_box_reg: 0.849  loss_mask: 0.637  loss_rpn_cls: 0.030  loss_rpn_loc: 0.009  time: 0.9363  data_time: 0.0095  lr: 0.000035  max_mem: 5730M\n",
      "\u001b[32m[04/19 05:14:44 d2.utils.events]: \u001b[0m eta: 0:36:31  iter: 159  total_loss: 2.233  loss_cls: 0.666  loss_box_reg: 0.896  loss_mask: 0.622  loss_rpn_cls: 0.030  loss_rpn_loc: 0.009  time: 0.9372  data_time: 0.0093  lr: 0.000040  max_mem: 5730M\n",
      "\u001b[32m[04/19 05:15:02 d2.utils.events]: \u001b[0m eta: 0:36:13  iter: 179  total_loss: 2.069  loss_cls: 0.591  loss_box_reg: 0.831  loss_mask: 0.603  loss_rpn_cls: 0.017  loss_rpn_loc: 0.008  time: 0.9379  data_time: 0.0090  lr: 0.000045  max_mem: 5730M\n",
      "\u001b[32m[04/19 05:15:21 d2.utils.events]: \u001b[0m eta: 0:35:56  iter: 199  total_loss: 2.109  loss_cls: 0.601  loss_box_reg: 0.872  loss_mask: 0.590  loss_rpn_cls: 0.015  loss_rpn_loc: 0.008  time: 0.9385  data_time: 0.0099  lr: 0.000050  max_mem: 5730M\n",
      "\u001b[32m[04/19 05:15:40 d2.utils.events]: \u001b[0m eta: 0:35:39  iter: 219  total_loss: 1.988  loss_cls: 0.577  loss_box_reg: 0.823  loss_mask: 0.561  loss_rpn_cls: 0.010  loss_rpn_loc: 0.007  time: 0.9388  data_time: 0.0097  lr: 0.000055  max_mem: 5730M\n",
      "\u001b[32m[04/19 05:15:59 d2.utils.events]: \u001b[0m eta: 0:35:22  iter: 239  total_loss: 1.959  loss_cls: 0.550  loss_box_reg: 0.829  loss_mask: 0.533  loss_rpn_cls: 0.031  loss_rpn_loc: 0.009  time: 0.9384  data_time: 0.0169  lr: 0.000060  max_mem: 5730M\n",
      "\u001b[32m[04/19 05:16:18 d2.utils.events]: \u001b[0m eta: 0:35:03  iter: 259  total_loss: 1.936  loss_cls: 0.529  loss_box_reg: 0.810  loss_mask: 0.510  loss_rpn_cls: 0.013  loss_rpn_loc: 0.009  time: 0.9386  data_time: 0.0095  lr: 0.000065  max_mem: 5730M\n",
      "\u001b[32m[04/19 05:16:37 d2.utils.events]: \u001b[0m eta: 0:34:45  iter: 279  total_loss: 1.803  loss_cls: 0.491  loss_box_reg: 0.774  loss_mask: 0.480  loss_rpn_cls: 0.019  loss_rpn_loc: 0.008  time: 0.9398  data_time: 0.0093  lr: 0.000070  max_mem: 5730M\n",
      "\u001b[32m[04/19 05:16:56 d2.utils.events]: \u001b[0m eta: 0:34:27  iter: 299  total_loss: 1.790  loss_cls: 0.507  loss_box_reg: 0.774  loss_mask: 0.454  loss_rpn_cls: 0.011  loss_rpn_loc: 0.009  time: 0.9402  data_time: 0.0088  lr: 0.000075  max_mem: 5774M\n",
      "\u001b[32m[04/19 05:17:15 d2.utils.events]: \u001b[0m eta: 0:34:08  iter: 319  total_loss: 1.744  loss_cls: 0.509  loss_box_reg: 0.755  loss_mask: 0.418  loss_rpn_cls: 0.014  loss_rpn_loc: 0.009  time: 0.9401  data_time: 0.0103  lr: 0.000080  max_mem: 5774M\n",
      "\u001b[32m[04/19 05:17:34 d2.utils.events]: \u001b[0m eta: 0:33:50  iter: 339  total_loss: 1.672  loss_cls: 0.488  loss_box_reg: 0.746  loss_mask: 0.413  loss_rpn_cls: 0.009  loss_rpn_loc: 0.009  time: 0.9403  data_time: 0.0121  lr: 0.000085  max_mem: 5774M\n",
      "\u001b[32m[04/19 05:17:53 d2.utils.events]: \u001b[0m eta: 0:33:31  iter: 359  total_loss: 1.647  loss_cls: 0.475  loss_box_reg: 0.701  loss_mask: 0.381  loss_rpn_cls: 0.022  loss_rpn_loc: 0.010  time: 0.9407  data_time: 0.0098  lr: 0.000090  max_mem: 5976M\n",
      "\u001b[32m[04/19 05:18:12 d2.utils.events]: \u001b[0m eta: 0:33:12  iter: 379  total_loss: 1.551  loss_cls: 0.456  loss_box_reg: 0.699  loss_mask: 0.364  loss_rpn_cls: 0.010  loss_rpn_loc: 0.011  time: 0.9406  data_time: 0.0101  lr: 0.000095  max_mem: 5976M\n",
      "\u001b[32m[04/19 05:18:31 d2.utils.events]: \u001b[0m eta: 0:32:53  iter: 399  total_loss: 1.476  loss_cls: 0.458  loss_box_reg: 0.625  loss_mask: 0.335  loss_rpn_cls: 0.006  loss_rpn_loc: 0.010  time: 0.9405  data_time: 0.0099  lr: 0.000100  max_mem: 5976M\n",
      "警告: せん断変形の結果、有効な領域が残らなかった\n",
      "\u001b[32m[04/19 05:18:49 d2.utils.events]: \u001b[0m eta: 0:32:34  iter: 419  total_loss: 1.430  loss_cls: 0.430  loss_box_reg: 0.619  loss_mask: 0.332  loss_rpn_cls: 0.008  loss_rpn_loc: 0.011  time: 0.9403  data_time: 0.0105  lr: 0.000105  max_mem: 5976M\n",
      "\u001b[32m[04/19 05:19:08 d2.utils.events]: \u001b[0m eta: 0:32:16  iter: 439  total_loss: 1.374  loss_cls: 0.426  loss_box_reg: 0.581  loss_mask: 0.324  loss_rpn_cls: 0.017  loss_rpn_loc: 0.013  time: 0.9406  data_time: 0.0094  lr: 0.000110  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:19:27 d2.utils.events]: \u001b[0m eta: 0:31:57  iter: 459  total_loss: 1.324  loss_cls: 0.452  loss_box_reg: 0.533  loss_mask: 0.315  loss_rpn_cls: 0.009  loss_rpn_loc: 0.011  time: 0.9402  data_time: 0.0095  lr: 0.000115  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:19:46 d2.utils.events]: \u001b[0m eta: 0:31:38  iter: 479  total_loss: 1.290  loss_cls: 0.402  loss_box_reg: 0.527  loss_mask: 0.298  loss_rpn_cls: 0.008  loss_rpn_loc: 0.010  time: 0.9405  data_time: 0.0103  lr: 0.000120  max_mem: 6036M\n",
      "警告: せん断変形の結果、有効な領域が残らなかった\n",
      "\u001b[32m[04/19 05:20:05 d2.utils.events]: \u001b[0m eta: 0:31:20  iter: 499  total_loss: 1.174  loss_cls: 0.386  loss_box_reg: 0.471  loss_mask: 0.268  loss_rpn_cls: 0.008  loss_rpn_loc: 0.010  time: 0.9404  data_time: 0.0121  lr: 0.000125  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:20:24 d2.utils.events]: \u001b[0m eta: 0:31:01  iter: 519  total_loss: 1.177  loss_cls: 0.388  loss_box_reg: 0.476  loss_mask: 0.294  loss_rpn_cls: 0.006  loss_rpn_loc: 0.008  time: 0.9402  data_time: 0.0092  lr: 0.000130  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:20:42 d2.utils.events]: \u001b[0m eta: 0:30:42  iter: 539  total_loss: 1.148  loss_cls: 0.385  loss_box_reg: 0.451  loss_mask: 0.273  loss_rpn_cls: 0.008  loss_rpn_loc: 0.010  time: 0.9400  data_time: 0.0099  lr: 0.000135  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:21:01 d2.utils.events]: \u001b[0m eta: 0:30:23  iter: 559  total_loss: 1.118  loss_cls: 0.354  loss_box_reg: 0.467  loss_mask: 0.284  loss_rpn_cls: 0.009  loss_rpn_loc: 0.009  time: 0.9401  data_time: 0.0118  lr: 0.000140  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:21:20 d2.utils.events]: \u001b[0m eta: 0:30:05  iter: 579  total_loss: 1.060  loss_cls: 0.325  loss_box_reg: 0.401  loss_mask: 0.266  loss_rpn_cls: 0.005  loss_rpn_loc: 0.009  time: 0.9401  data_time: 0.0091  lr: 0.000145  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:21:39 d2.utils.events]: \u001b[0m eta: 0:29:46  iter: 599  total_loss: 1.006  loss_cls: 0.338  loss_box_reg: 0.395  loss_mask: 0.244  loss_rpn_cls: 0.008  loss_rpn_loc: 0.009  time: 0.9401  data_time: 0.0106  lr: 0.000150  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:21:58 d2.utils.events]: \u001b[0m eta: 0:29:27  iter: 619  total_loss: 0.977  loss_cls: 0.318  loss_box_reg: 0.370  loss_mask: 0.246  loss_rpn_cls: 0.005  loss_rpn_loc: 0.007  time: 0.9399  data_time: 0.0102  lr: 0.000155  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:22:17 d2.utils.events]: \u001b[0m eta: 0:29:09  iter: 639  total_loss: 0.950  loss_cls: 0.294  loss_box_reg: 0.395  loss_mask: 0.254  loss_rpn_cls: 0.005  loss_rpn_loc: 0.010  time: 0.9401  data_time: 0.0167  lr: 0.000160  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:22:35 d2.utils.events]: \u001b[0m eta: 0:28:50  iter: 659  total_loss: 0.999  loss_cls: 0.277  loss_box_reg: 0.384  loss_mask: 0.251  loss_rpn_cls: 0.007  loss_rpn_loc: 0.009  time: 0.9400  data_time: 0.0099  lr: 0.000165  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:22:54 d2.utils.events]: \u001b[0m eta: 0:28:31  iter: 679  total_loss: 0.880  loss_cls: 0.257  loss_box_reg: 0.329  loss_mask: 0.259  loss_rpn_cls: 0.006  loss_rpn_loc: 0.008  time: 0.9399  data_time: 0.0102  lr: 0.000170  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:23:13 d2.utils.events]: \u001b[0m eta: 0:28:12  iter: 699  total_loss: 0.905  loss_cls: 0.258  loss_box_reg: 0.363  loss_mask: 0.240  loss_rpn_cls: 0.006  loss_rpn_loc: 0.009  time: 0.9398  data_time: 0.0092  lr: 0.000175  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:23:32 d2.utils.events]: \u001b[0m eta: 0:27:53  iter: 719  total_loss: 0.825  loss_cls: 0.233  loss_box_reg: 0.307  loss_mask: 0.226  loss_rpn_cls: 0.006  loss_rpn_loc: 0.008  time: 0.9398  data_time: 0.0178  lr: 0.000180  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:23:51 d2.utils.events]: \u001b[0m eta: 0:27:35  iter: 739  total_loss: 0.808  loss_cls: 0.247  loss_box_reg: 0.333  loss_mask: 0.221  loss_rpn_cls: 0.004  loss_rpn_loc: 0.008  time: 0.9399  data_time: 0.0098  lr: 0.000185  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:24:10 d2.utils.events]: \u001b[0m eta: 0:27:16  iter: 759  total_loss: 0.764  loss_cls: 0.223  loss_box_reg: 0.291  loss_mask: 0.208  loss_rpn_cls: 0.006  loss_rpn_loc: 0.007  time: 0.9398  data_time: 0.0181  lr: 0.000190  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:24:28 d2.utils.events]: \u001b[0m eta: 0:26:57  iter: 779  total_loss: 0.681  loss_cls: 0.174  loss_box_reg: 0.270  loss_mask: 0.216  loss_rpn_cls: 0.004  loss_rpn_loc: 0.007  time: 0.9398  data_time: 0.0097  lr: 0.000195  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:24:47 d2.utils.events]: \u001b[0m eta: 0:26:38  iter: 799  total_loss: 0.709  loss_cls: 0.188  loss_box_reg: 0.270  loss_mask: 0.206  loss_rpn_cls: 0.003  loss_rpn_loc: 0.007  time: 0.9399  data_time: 0.0093  lr: 0.000200  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:25:06 d2.utils.events]: \u001b[0m eta: 0:26:19  iter: 819  total_loss: 0.667  loss_cls: 0.186  loss_box_reg: 0.282  loss_mask: 0.202  loss_rpn_cls: 0.004  loss_rpn_loc: 0.008  time: 0.9398  data_time: 0.0092  lr: 0.000205  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:25:25 d2.utils.events]: \u001b[0m eta: 0:26:00  iter: 839  total_loss: 0.640  loss_cls: 0.175  loss_box_reg: 0.256  loss_mask: 0.196  loss_rpn_cls: 0.003  loss_rpn_loc: 0.007  time: 0.9398  data_time: 0.0089  lr: 0.000210  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:25:44 d2.utils.events]: \u001b[0m eta: 0:25:41  iter: 859  total_loss: 0.648  loss_cls: 0.162  loss_box_reg: 0.273  loss_mask: 0.202  loss_rpn_cls: 0.003  loss_rpn_loc: 0.007  time: 0.9397  data_time: 0.0107  lr: 0.000215  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:26:02 d2.utils.events]: \u001b[0m eta: 0:25:23  iter: 879  total_loss: 0.606  loss_cls: 0.150  loss_box_reg: 0.253  loss_mask: 0.195  loss_rpn_cls: 0.004  loss_rpn_loc: 0.007  time: 0.9395  data_time: 0.0093  lr: 0.000220  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:26:21 d2.utils.events]: \u001b[0m eta: 0:25:04  iter: 899  total_loss: 0.620  loss_cls: 0.170  loss_box_reg: 0.248  loss_mask: 0.193  loss_rpn_cls: 0.005  loss_rpn_loc: 0.007  time: 0.9394  data_time: 0.0087  lr: 0.000225  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:26:40 d2.utils.events]: \u001b[0m eta: 0:24:45  iter: 919  total_loss: 0.649  loss_cls: 0.158  loss_box_reg: 0.263  loss_mask: 0.198  loss_rpn_cls: 0.004  loss_rpn_loc: 0.007  time: 0.9396  data_time: 0.0120  lr: 0.000230  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:26:59 d2.utils.events]: \u001b[0m eta: 0:24:26  iter: 939  total_loss: 0.603  loss_cls: 0.126  loss_box_reg: 0.235  loss_mask: 0.195  loss_rpn_cls: 0.003  loss_rpn_loc: 0.006  time: 0.9395  data_time: 0.0102  lr: 0.000235  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:27:18 d2.utils.events]: \u001b[0m eta: 0:24:07  iter: 959  total_loss: 0.534  loss_cls: 0.116  loss_box_reg: 0.212  loss_mask: 0.191  loss_rpn_cls: 0.007  loss_rpn_loc: 0.006  time: 0.9394  data_time: 0.0101  lr: 0.000240  max_mem: 6036M\n",
      "警告: せん断変形の結果、有効な領域が残らなかった\n",
      "\u001b[32m[04/19 05:27:36 d2.utils.events]: \u001b[0m eta: 0:23:48  iter: 979  total_loss: 0.599  loss_cls: 0.136  loss_box_reg: 0.253  loss_mask: 0.206  loss_rpn_cls: 0.004  loss_rpn_loc: 0.006  time: 0.9392  data_time: 0.0096  lr: 0.000245  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:27:55 d2.utils.events]: \u001b[0m eta: 0:23:29  iter: 999  total_loss: 0.526  loss_cls: 0.113  loss_box_reg: 0.229  loss_mask: 0.176  loss_rpn_cls: 0.003  loss_rpn_loc: 0.006  time: 0.9389  data_time: 0.0091  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:28:14 d2.utils.events]: \u001b[0m eta: 0:23:10  iter: 1019  total_loss: 0.548  loss_cls: 0.115  loss_box_reg: 0.220  loss_mask: 0.185  loss_rpn_cls: 0.002  loss_rpn_loc: 0.006  time: 0.9389  data_time: 0.0109  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:28:32 d2.utils.events]: \u001b[0m eta: 0:22:52  iter: 1039  total_loss: 0.555  loss_cls: 0.113  loss_box_reg: 0.236  loss_mask: 0.188  loss_rpn_cls: 0.004  loss_rpn_loc: 0.007  time: 0.9388  data_time: 0.0092  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:28:51 d2.utils.events]: \u001b[0m eta: 0:22:33  iter: 1059  total_loss: 0.597  loss_cls: 0.135  loss_box_reg: 0.231  loss_mask: 0.193  loss_rpn_cls: 0.004  loss_rpn_loc: 0.007  time: 0.9387  data_time: 0.0105  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:29:10 d2.utils.events]: \u001b[0m eta: 0:22:14  iter: 1079  total_loss: 0.471  loss_cls: 0.100  loss_box_reg: 0.212  loss_mask: 0.174  loss_rpn_cls: 0.002  loss_rpn_loc: 0.006  time: 0.9388  data_time: 0.0094  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:29:29 d2.utils.events]: \u001b[0m eta: 0:21:55  iter: 1099  total_loss: 0.533  loss_cls: 0.110  loss_box_reg: 0.233  loss_mask: 0.165  loss_rpn_cls: 0.003  loss_rpn_loc: 0.006  time: 0.9388  data_time: 0.0094  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:29:48 d2.utils.events]: \u001b[0m eta: 0:21:36  iter: 1119  total_loss: 0.549  loss_cls: 0.104  loss_box_reg: 0.227  loss_mask: 0.190  loss_rpn_cls: 0.002  loss_rpn_loc: 0.007  time: 0.9387  data_time: 0.0104  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:30:06 d2.utils.events]: \u001b[0m eta: 0:21:17  iter: 1139  total_loss: 0.518  loss_cls: 0.097  loss_box_reg: 0.219  loss_mask: 0.175  loss_rpn_cls: 0.002  loss_rpn_loc: 0.006  time: 0.9386  data_time: 0.0098  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:30:25 d2.utils.events]: \u001b[0m eta: 0:20:58  iter: 1159  total_loss: 0.494  loss_cls: 0.078  loss_box_reg: 0.202  loss_mask: 0.180  loss_rpn_cls: 0.001  loss_rpn_loc: 0.007  time: 0.9385  data_time: 0.0095  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:30:44 d2.utils.events]: \u001b[0m eta: 0:20:39  iter: 1179  total_loss: 0.460  loss_cls: 0.069  loss_box_reg: 0.198  loss_mask: 0.170  loss_rpn_cls: 0.002  loss_rpn_loc: 0.005  time: 0.9383  data_time: 0.0095  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:31:03 d2.utils.events]: \u001b[0m eta: 0:20:20  iter: 1199  total_loss: 0.532  loss_cls: 0.097  loss_box_reg: 0.225  loss_mask: 0.173  loss_rpn_cls: 0.003  loss_rpn_loc: 0.007  time: 0.9383  data_time: 0.0096  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:31:21 d2.utils.events]: \u001b[0m eta: 0:20:01  iter: 1219  total_loss: 0.546  loss_cls: 0.098  loss_box_reg: 0.227  loss_mask: 0.187  loss_rpn_cls: 0.003  loss_rpn_loc: 0.006  time: 0.9384  data_time: 0.0089  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:31:40 d2.utils.events]: \u001b[0m eta: 0:19:42  iter: 1239  total_loss: 0.466  loss_cls: 0.091  loss_box_reg: 0.199  loss_mask: 0.152  loss_rpn_cls: 0.003  loss_rpn_loc: 0.006  time: 0.9383  data_time: 0.0091  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:31:59 d2.utils.events]: \u001b[0m eta: 0:19:24  iter: 1259  total_loss: 0.461  loss_cls: 0.080  loss_box_reg: 0.205  loss_mask: 0.161  loss_rpn_cls: 0.002  loss_rpn_loc: 0.005  time: 0.9383  data_time: 0.0093  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:32:18 d2.utils.events]: \u001b[0m eta: 0:19:05  iter: 1279  total_loss: 0.468  loss_cls: 0.089  loss_box_reg: 0.214  loss_mask: 0.159  loss_rpn_cls: 0.002  loss_rpn_loc: 0.006  time: 0.9383  data_time: 0.0092  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:32:37 d2.utils.events]: \u001b[0m eta: 0:18:46  iter: 1299  total_loss: 0.506  loss_cls: 0.074  loss_box_reg: 0.205  loss_mask: 0.156  loss_rpn_cls: 0.003  loss_rpn_loc: 0.006  time: 0.9383  data_time: 0.0095  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:32:55 d2.utils.events]: \u001b[0m eta: 0:18:27  iter: 1319  total_loss: 0.406  loss_cls: 0.061  loss_box_reg: 0.184  loss_mask: 0.168  loss_rpn_cls: 0.001  loss_rpn_loc: 0.006  time: 0.9383  data_time: 0.0117  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:33:14 d2.utils.events]: \u001b[0m eta: 0:18:08  iter: 1339  total_loss: 0.471  loss_cls: 0.088  loss_box_reg: 0.193  loss_mask: 0.181  loss_rpn_cls: 0.003  loss_rpn_loc: 0.006  time: 0.9383  data_time: 0.0101  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:33:33 d2.utils.events]: \u001b[0m eta: 0:17:49  iter: 1359  total_loss: 0.456  loss_cls: 0.065  loss_box_reg: 0.199  loss_mask: 0.163  loss_rpn_cls: 0.004  loss_rpn_loc: 0.007  time: 0.9382  data_time: 0.0134  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:33:52 d2.utils.events]: \u001b[0m eta: 0:17:30  iter: 1379  total_loss: 0.436  loss_cls: 0.073  loss_box_reg: 0.198  loss_mask: 0.169  loss_rpn_cls: 0.002  loss_rpn_loc: 0.005  time: 0.9381  data_time: 0.0093  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:34:10 d2.utils.events]: \u001b[0m eta: 0:17:11  iter: 1399  total_loss: 0.422  loss_cls: 0.075  loss_box_reg: 0.180  loss_mask: 0.173  loss_rpn_cls: 0.004  loss_rpn_loc: 0.006  time: 0.9381  data_time: 0.0103  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:34:29 d2.utils.events]: \u001b[0m eta: 0:16:53  iter: 1419  total_loss: 0.459  loss_cls: 0.081  loss_box_reg: 0.183  loss_mask: 0.161  loss_rpn_cls: 0.003  loss_rpn_loc: 0.005  time: 0.9380  data_time: 0.0102  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:34:48 d2.utils.events]: \u001b[0m eta: 0:16:34  iter: 1439  total_loss: 0.405  loss_cls: 0.055  loss_box_reg: 0.178  loss_mask: 0.156  loss_rpn_cls: 0.002  loss_rpn_loc: 0.006  time: 0.9379  data_time: 0.0103  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:35:07 d2.utils.events]: \u001b[0m eta: 0:16:15  iter: 1459  total_loss: 0.448  loss_cls: 0.087  loss_box_reg: 0.195  loss_mask: 0.166  loss_rpn_cls: 0.002  loss_rpn_loc: 0.005  time: 0.9381  data_time: 0.0116  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:35:26 d2.utils.events]: \u001b[0m eta: 0:15:56  iter: 1479  total_loss: 0.395  loss_cls: 0.054  loss_box_reg: 0.179  loss_mask: 0.153  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9380  data_time: 0.0095  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:35:44 d2.utils.events]: \u001b[0m eta: 0:15:37  iter: 1499  total_loss: 0.416  loss_cls: 0.066  loss_box_reg: 0.183  loss_mask: 0.171  loss_rpn_cls: 0.002  loss_rpn_loc: 0.007  time: 0.9379  data_time: 0.0155  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:36:03 d2.utils.events]: \u001b[0m eta: 0:15:19  iter: 1519  total_loss: 0.401  loss_cls: 0.065  loss_box_reg: 0.174  loss_mask: 0.162  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9379  data_time: 0.0108  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:36:22 d2.utils.events]: \u001b[0m eta: 0:15:00  iter: 1539  total_loss: 0.397  loss_cls: 0.057  loss_box_reg: 0.155  loss_mask: 0.141  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9380  data_time: 0.0102  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:36:41 d2.utils.events]: \u001b[0m eta: 0:14:41  iter: 1559  total_loss: 0.381  loss_cls: 0.058  loss_box_reg: 0.159  loss_mask: 0.154  loss_rpn_cls: 0.002  loss_rpn_loc: 0.006  time: 0.9381  data_time: 0.0093  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:37:00 d2.utils.events]: \u001b[0m eta: 0:14:22  iter: 1579  total_loss: 0.399  loss_cls: 0.052  loss_box_reg: 0.172  loss_mask: 0.156  loss_rpn_cls: 0.002  loss_rpn_loc: 0.006  time: 0.9380  data_time: 0.0095  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:37:18 d2.utils.events]: \u001b[0m eta: 0:14:03  iter: 1599  total_loss: 0.371  loss_cls: 0.055  loss_box_reg: 0.168  loss_mask: 0.147  loss_rpn_cls: 0.001  loss_rpn_loc: 0.006  time: 0.9380  data_time: 0.0098  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:37:37 d2.utils.events]: \u001b[0m eta: 0:13:45  iter: 1619  total_loss: 0.397  loss_cls: 0.051  loss_box_reg: 0.171  loss_mask: 0.149  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9379  data_time: 0.0091  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:37:56 d2.utils.events]: \u001b[0m eta: 0:13:26  iter: 1639  total_loss: 0.423  loss_cls: 0.062  loss_box_reg: 0.175  loss_mask: 0.160  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9378  data_time: 0.0093  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:38:15 d2.utils.events]: \u001b[0m eta: 0:13:07  iter: 1659  total_loss: 0.382  loss_cls: 0.052  loss_box_reg: 0.168  loss_mask: 0.139  loss_rpn_cls: 0.001  loss_rpn_loc: 0.006  time: 0.9378  data_time: 0.0108  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:38:33 d2.utils.events]: \u001b[0m eta: 0:12:48  iter: 1679  total_loss: 0.430  loss_cls: 0.075  loss_box_reg: 0.178  loss_mask: 0.164  loss_rpn_cls: 0.001  loss_rpn_loc: 0.006  time: 0.9378  data_time: 0.0092  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:38:52 d2.utils.events]: \u001b[0m eta: 0:12:29  iter: 1699  total_loss: 0.371  loss_cls: 0.055  loss_box_reg: 0.178  loss_mask: 0.135  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9378  data_time: 0.0122  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:39:11 d2.utils.events]: \u001b[0m eta: 0:12:11  iter: 1719  total_loss: 0.380  loss_cls: 0.056  loss_box_reg: 0.172  loss_mask: 0.153  loss_rpn_cls: 0.002  loss_rpn_loc: 0.005  time: 0.9378  data_time: 0.0095  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:39:30 d2.utils.events]: \u001b[0m eta: 0:11:52  iter: 1739  total_loss: 0.384  loss_cls: 0.050  loss_box_reg: 0.154  loss_mask: 0.160  loss_rpn_cls: 0.002  loss_rpn_loc: 0.005  time: 0.9378  data_time: 0.0089  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:39:49 d2.utils.events]: \u001b[0m eta: 0:11:33  iter: 1759  total_loss: 0.402  loss_cls: 0.057  loss_box_reg: 0.167  loss_mask: 0.158  loss_rpn_cls: 0.002  loss_rpn_loc: 0.005  time: 0.9378  data_time: 0.0096  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:40:07 d2.utils.events]: \u001b[0m eta: 0:11:14  iter: 1779  total_loss: 0.407  loss_cls: 0.064  loss_box_reg: 0.178  loss_mask: 0.156  loss_rpn_cls: 0.002  loss_rpn_loc: 0.006  time: 0.9378  data_time: 0.0092  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:40:26 d2.utils.events]: \u001b[0m eta: 0:10:55  iter: 1799  total_loss: 0.392  loss_cls: 0.068  loss_box_reg: 0.184  loss_mask: 0.141  loss_rpn_cls: 0.003  loss_rpn_loc: 0.006  time: 0.9377  data_time: 0.0098  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:40:45 d2.utils.events]: \u001b[0m eta: 0:10:37  iter: 1819  total_loss: 0.380  loss_cls: 0.058  loss_box_reg: 0.162  loss_mask: 0.152  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9377  data_time: 0.0115  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:41:04 d2.utils.events]: \u001b[0m eta: 0:10:18  iter: 1839  total_loss: 0.376  loss_cls: 0.061  loss_box_reg: 0.155  loss_mask: 0.143  loss_rpn_cls: 0.002  loss_rpn_loc: 0.005  time: 0.9377  data_time: 0.0099  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:41:22 d2.utils.events]: \u001b[0m eta: 0:09:59  iter: 1859  total_loss: 0.335  loss_cls: 0.042  loss_box_reg: 0.146  loss_mask: 0.149  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9376  data_time: 0.0098  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:41:41 d2.utils.events]: \u001b[0m eta: 0:09:40  iter: 1879  total_loss: 0.378  loss_cls: 0.050  loss_box_reg: 0.161  loss_mask: 0.149  loss_rpn_cls: 0.002  loss_rpn_loc: 0.006  time: 0.9376  data_time: 0.0094  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:42:00 d2.utils.events]: \u001b[0m eta: 0:09:22  iter: 1899  total_loss: 0.392  loss_cls: 0.054  loss_box_reg: 0.166  loss_mask: 0.141  loss_rpn_cls: 0.002  loss_rpn_loc: 0.006  time: 0.9375  data_time: 0.0096  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:42:19 d2.utils.events]: \u001b[0m eta: 0:09:03  iter: 1919  total_loss: 0.357  loss_cls: 0.041  loss_box_reg: 0.157  loss_mask: 0.142  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9375  data_time: 0.0088  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:42:37 d2.utils.events]: \u001b[0m eta: 0:08:44  iter: 1939  total_loss: 0.321  loss_cls: 0.043  loss_box_reg: 0.149  loss_mask: 0.132  loss_rpn_cls: 0.001  loss_rpn_loc: 0.004  time: 0.9375  data_time: 0.0089  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:42:56 d2.utils.events]: \u001b[0m eta: 0:08:25  iter: 1959  total_loss: 0.338  loss_cls: 0.052  loss_box_reg: 0.150  loss_mask: 0.133  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9374  data_time: 0.0120  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:43:15 d2.utils.events]: \u001b[0m eta: 0:08:07  iter: 1979  total_loss: 0.339  loss_cls: 0.044  loss_box_reg: 0.154  loss_mask: 0.145  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9375  data_time: 0.0095  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:43:33 d2.utils.events]: \u001b[0m eta: 0:07:48  iter: 1999  total_loss: 0.364  loss_cls: 0.049  loss_box_reg: 0.165  loss_mask: 0.131  loss_rpn_cls: 0.001  loss_rpn_loc: 0.006  time: 0.9374  data_time: 0.0205  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:43:52 d2.utils.events]: \u001b[0m eta: 0:07:29  iter: 2019  total_loss: 0.327  loss_cls: 0.045  loss_box_reg: 0.153  loss_mask: 0.132  loss_rpn_cls: 0.002  loss_rpn_loc: 0.005  time: 0.9374  data_time: 0.0090  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:44:11 d2.utils.events]: \u001b[0m eta: 0:07:11  iter: 2039  total_loss: 0.403  loss_cls: 0.042  loss_box_reg: 0.173  loss_mask: 0.147  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9374  data_time: 0.0100  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:44:30 d2.utils.events]: \u001b[0m eta: 0:06:52  iter: 2059  total_loss: 0.335  loss_cls: 0.046  loss_box_reg: 0.151  loss_mask: 0.131  loss_rpn_cls: 0.001  loss_rpn_loc: 0.004  time: 0.9374  data_time: 0.0097  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:44:49 d2.utils.events]: \u001b[0m eta: 0:06:33  iter: 2079  total_loss: 0.333  loss_cls: 0.049  loss_box_reg: 0.146  loss_mask: 0.132  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9373  data_time: 0.0109  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:45:07 d2.utils.events]: \u001b[0m eta: 0:06:15  iter: 2099  total_loss: 0.336  loss_cls: 0.040  loss_box_reg: 0.151  loss_mask: 0.139  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9373  data_time: 0.0093  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:45:26 d2.utils.events]: \u001b[0m eta: 0:05:56  iter: 2119  total_loss: 0.347  loss_cls: 0.041  loss_box_reg: 0.144  loss_mask: 0.134  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9373  data_time: 0.0095  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:45:45 d2.utils.events]: \u001b[0m eta: 0:05:37  iter: 2139  total_loss: 0.350  loss_cls: 0.051  loss_box_reg: 0.164  loss_mask: 0.146  loss_rpn_cls: 0.001  loss_rpn_loc: 0.004  time: 0.9372  data_time: 0.0096  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:46:03 d2.utils.events]: \u001b[0m eta: 0:05:18  iter: 2159  total_loss: 0.322  loss_cls: 0.035  loss_box_reg: 0.142  loss_mask: 0.132  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9372  data_time: 0.0091  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:46:22 d2.utils.events]: \u001b[0m eta: 0:05:00  iter: 2179  total_loss: 0.365  loss_cls: 0.051  loss_box_reg: 0.164  loss_mask: 0.141  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9371  data_time: 0.0169  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:46:41 d2.utils.events]: \u001b[0m eta: 0:04:41  iter: 2199  total_loss: 0.330  loss_cls: 0.047  loss_box_reg: 0.152  loss_mask: 0.131  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9370  data_time: 0.0100  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:47:00 d2.utils.events]: \u001b[0m eta: 0:04:22  iter: 2219  total_loss: 0.351  loss_cls: 0.041  loss_box_reg: 0.141  loss_mask: 0.137  loss_rpn_cls: 0.001  loss_rpn_loc: 0.004  time: 0.9370  data_time: 0.0089  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:47:18 d2.utils.events]: \u001b[0m eta: 0:04:04  iter: 2239  total_loss: 0.326  loss_cls: 0.047  loss_box_reg: 0.132  loss_mask: 0.132  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9370  data_time: 0.0118  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:47:37 d2.utils.events]: \u001b[0m eta: 0:03:45  iter: 2259  total_loss: 0.341  loss_cls: 0.044  loss_box_reg: 0.152  loss_mask: 0.143  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9369  data_time: 0.0131  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:47:56 d2.utils.events]: \u001b[0m eta: 0:03:26  iter: 2279  total_loss: 0.341  loss_cls: 0.052  loss_box_reg: 0.156  loss_mask: 0.141  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9369  data_time: 0.0097  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:48:14 d2.utils.events]: \u001b[0m eta: 0:03:07  iter: 2299  total_loss: 0.343  loss_cls: 0.042  loss_box_reg: 0.135  loss_mask: 0.133  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9368  data_time: 0.0096  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:48:33 d2.utils.events]: \u001b[0m eta: 0:02:49  iter: 2319  total_loss: 0.342  loss_cls: 0.046  loss_box_reg: 0.150  loss_mask: 0.144  loss_rpn_cls: 0.002  loss_rpn_loc: 0.004  time: 0.9369  data_time: 0.0182  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:48:52 d2.utils.events]: \u001b[0m eta: 0:02:30  iter: 2339  total_loss: 0.327  loss_cls: 0.050  loss_box_reg: 0.140  loss_mask: 0.127  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9369  data_time: 0.0096  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:49:11 d2.utils.events]: \u001b[0m eta: 0:02:11  iter: 2359  total_loss: 0.342  loss_cls: 0.047  loss_box_reg: 0.158  loss_mask: 0.129  loss_rpn_cls: 0.002  loss_rpn_loc: 0.006  time: 0.9370  data_time: 0.0094  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:49:30 d2.utils.events]: \u001b[0m eta: 0:01:53  iter: 2379  total_loss: 0.344  loss_cls: 0.062  loss_box_reg: 0.152  loss_mask: 0.140  loss_rpn_cls: 0.002  loss_rpn_loc: 0.006  time: 0.9369  data_time: 0.0138  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:49:48 d2.utils.events]: \u001b[0m eta: 0:01:34  iter: 2399  total_loss: 0.324  loss_cls: 0.050  loss_box_reg: 0.143  loss_mask: 0.132  loss_rpn_cls: 0.001  loss_rpn_loc: 0.006  time: 0.9369  data_time: 0.0095  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:50:07 d2.utils.events]: \u001b[0m eta: 0:01:15  iter: 2419  total_loss: 0.318  loss_cls: 0.047  loss_box_reg: 0.121  loss_mask: 0.139  loss_rpn_cls: 0.002  loss_rpn_loc: 0.005  time: 0.9369  data_time: 0.0092  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:50:26 d2.utils.events]: \u001b[0m eta: 0:00:57  iter: 2439  total_loss: 0.307  loss_cls: 0.030  loss_box_reg: 0.147  loss_mask: 0.132  loss_rpn_cls: 0.001  loss_rpn_loc: 0.004  time: 0.9369  data_time: 0.0091  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:50:45 d2.utils.events]: \u001b[0m eta: 0:00:38  iter: 2459  total_loss: 0.345  loss_cls: 0.044  loss_box_reg: 0.152  loss_mask: 0.141  loss_rpn_cls: 0.000  loss_rpn_loc: 0.005  time: 0.9368  data_time: 0.0095  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:51:03 d2.utils.events]: \u001b[0m eta: 0:00:19  iter: 2479  total_loss: 0.340  loss_cls: 0.042  loss_box_reg: 0.152  loss_mask: 0.129  loss_rpn_cls: 0.001  loss_rpn_loc: 0.005  time: 0.9368  data_time: 0.0096  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:51:30 d2.utils.events]: \u001b[0m eta: 0:00:00  iter: 2499  total_loss: 0.311  loss_cls: 0.036  loss_box_reg: 0.137  loss_mask: 0.130  loss_rpn_cls: 0.000  loss_rpn_loc: 0.005  time: 0.9368  data_time: 0.0096  lr: 0.000250  max_mem: 6036M\n",
      "\u001b[32m[04/19 05:51:31 d2.engine.hooks]: \u001b[0mOverall training speed: 2497 iterations in 0:39:00 (0.9372 s / it)\n",
      "\u001b[32m[04/19 05:51:31 d2.engine.hooks]: \u001b[0mTotal training time: 0:39:15 (0:00:15 on hooks)\n"
     ]
    }
   ],
   "source": [
    "from detectron2.data import build_detection_test_loader, build_detection_train_loader\n",
    "\n",
    "# 出力先のディレクトリを作る\n",
    "os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\n",
    "\n",
    "trainer = MachikadoTrainer(cfg) \n",
    "trainer.resume_or_load(resume=False)\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}