Add confusion matrix eval tool and extend pptsm training/augmentation…

configs/recognition/pptsm/pptsm_k400_videos_uniform.yaml

@@ -2,15 +2,21 @@ MODEL: #MODEL field
     framework: "Recognizer2D" #Mandatory, indicate the type of network, associate to the 'paddlevideo/modeling/framework/' .
     backbone: #Mandatory, indicate the type of backbone, associate to the 'paddlevideo/modeling/backbones/' .
         name: "ResNetTweaksTSM" #Mandatory, The name of backbone.
-        pretrained: "data/ResNet50_vd_ssld_v2_pretrained.pdparams" #Optional, pretrained model path.
-        depth: 50 #Optional, the depth of backbone architecture.
+        pretrained: "data/ResNet101_vd_ssld_pretrained.pdparams" #Optional, pretrained model path.
+        depth: 101 #Optional, the depth of backbone architecture.
     head:
         name: "ppTSMHead" #Mandatory, indicate the type of head, associate to the 'paddlevideo/modeling/heads'
-        num_classes: 400 #Optional, the number of classes to be classified.
+        num_classes: 7 #Optional, the number of classes to be classified.
         in_channels: 2048 #input channel of the extracted feature.
         drop_ratio: 0.5 #the ratio of dropout
         std: 0.01 #std value in params initialization
         ls_eps: 0.1 # label smooth factor
+        loss_cfg:
+            name: "CrossEntropyLoss"
+            # per-class weight to counter label imbalance (inverse train-set
+            # frequency, normalized to mean 1), order matches label index:
+            # [Arrest, Arson, Explosion, Fighting, Normal, RoadAccident, Vandalism]
+            class_weight: [2.6353, 0.4362, 2.6353, 0.1550, 0.0626, 0.3227, 0.7529]
 
 DATASET: #DATASET field
     batch_size: 16  #Mandatory, bacth size
@@ -119,6 +125,7 @@ INFERENCE:
     name: 'ppTSM_Inference_helper'
     num_seg: 8
     target_size: 224
+    top_k: 5
 
 model_name: "ppTSM"
 log_interval: 10 #Optional, the interal of logger, default:10

configs/recognition/pptsm/pptsm_mv2_k400_videos_uniform.yaml

@@ -3,24 +3,46 @@ MODEL: #MODEL field
     backbone:
         name: "PPTSM_MobileNetV2" #Mandatory, The name of backbone.
         pretrained: "data/MobileNetV2_ssld_pretrained.pdparams" #Optional, pretrained model path.
+        class_num: 4 #Optional, the number of classes to be classified. Defaults to 400 (Kinetics-400) if omitted.
+        num_seg: 16 #Must match PIPELINE.*.sample.num_seg exactly: the backbone
+        #reshapes its flattened [N*T,C,H,W] features using this fixed value at
+        #forward() time (see pptsm_mv2.py MobileNet.forward), it does NOT read
+        #num_seg dynamically from the data like ResNetTweaksTSM+ppTSMHead does.
+        #A mismatch silently scales the output batch dim and crashes the loss.
     head:
         name: "MoViNetHead" #Mandatory, indicate the type of head, associate to the 'paddlevideo/modeling/heads'
+        loss_cfg:
+            name: "CrossEntropyLoss"
+            # per-class weight to counter label imbalance (inverse train-set
+            # frequency, normalized to mean 1), order matches label index:
+            # [Arson, Fighting, Normal, RoadAccident]
+            class_weight: [1.7867, 0.6350, 0.2565, 1.3218]
 
 DATASET: #DATASET field
-    batch_size: 16  #Mandatory, bacth size
-    num_workers: 4 #Mandatory, the number of subprocess on each GPU.
+    batch_size: 16  #Mandatory, bacth size. Kept low relative to num_seg AND
+    #input resolution: this backbone flattens batch_size*num_seg frames into
+    #one Conv2D batch (pptsm_mv2.py MobileNet.forward), and cuDNN's int32
+    #output-tensor indexing overflows past a frame-count threshold that
+    #shrinks with resolution (each early Conv2D layer's element count scales
+    #with H*W). Going from 224 to 640 input (~8.16x more pixels/frame) shrinks
+    #that safe threshold by roughly the same factor, so batch_size had to come
+    #down from 64 to 8 to stay under it (8*16=128 frames/step). Tune cautiously
+    #from here — the exact threshold depends on cuDNN's internal algorithm
+    #padding, which isn't precisely predictable, and 640x640 activations will
+    #also cost far more raw GPU memory, so watch for CUDA OOM independently.
+    num_workers: 8 #Mandatory, the number of subprocess on each GPU.
     train:
         format: "VideoDataset" #Mandatory, indicate the type of dataset, associate to the 'paddlevidel/loader/dateset'
-        data_prefix: "data/k400" #Mandatory, train data root path
-        file_path: "data/k400/train.list" #Mandatory, train data index file path
+        data_prefix: "data/merge_030726/Train" #Mandatory, train data root path
+        file_path: "data/merge_030726/Train/train.txt" #Mandatory, train data index file path
     valid:
         format: "VideoDataset" #Mandatory, indicate the type of dataset, associate to the 'paddlevidel/loader/dateset'
-        data_prefix: "data/k400" #Mandatory, valid data root path
-        file_path: "data/k400/val.list" #Mandatory, valid data index file path
+        data_prefix: "data/merge_030726/Valid" #Mandatory, valid data root path
+        file_path: "data/merge_030726/Valid/val.txt" #Mandatory, valid data index file path
     test:
         format: "VideoDataset" #Mandatory, indicate the type of dataset, associate to the 'paddlevidel/loader/dateset'
-        data_prefix: "data/k400" #Mandatory, valid data root path
-        file_path: "data/k400/val.list" #Mandatory, valid data index file path
+        data_prefix: "data/merge_030726/Valid" #Mandatory, valid data root path
+        file_path: "data/merge_030726/Valid/val.txt" #Mandatory, valid data index file path
 
 PIPELINE: #PIPELINE field
     train: #Mandotary, indicate the pipeline to deal with the training data, associate to the 'paddlevideo/loader/pipelines/'
@@ -29,16 +51,12 @@ PIPELINE: #PIPELINE field
             backend: "decord"
         sample:
             name: "Sampler"
-            num_seg: 8
+            num_seg: 16
             seg_len: 1
             valid_mode: False
         transform: #Mandotary, image transfrom operator
-            - Scale:
-                short_size: 256
-            - MultiScaleCrop:
-                target_size: 256
-            - RandomCrop:
-                target_size: 224
+            - Letterbox:
+                target_size: [320, 180]
             - RandomFlip:
             - Image2Array:
             - Normalization:
@@ -50,14 +68,12 @@ PIPELINE: #PIPELINE field
             backend: "decord"
         sample:
             name: "Sampler"
-            num_seg: 8
+            num_seg: 16
             seg_len: 1
             valid_mode: True
         transform:
-            - Scale:
-                short_size: 256
-            - CenterCrop:
-                target_size: 224
+            - Letterbox:
+                target_size: [320, 180]
             - Image2Array:
             - Normalization:
                 mean: [0.485, 0.456, 0.406]
@@ -68,14 +84,12 @@ PIPELINE: #PIPELINE field
             backend: "decord"
         sample:
             name: "Sampler"
-            num_seg: 8
+            num_seg: 16
             seg_len: 1
             valid_mode: True
         transform:
-            - Scale:
-                short_size: 256
-            - CenterCrop:
-                target_size: 224
+            - Letterbox:
+                target_size: [320, 180]
             - Image2Array:
             - Normalization:
                 mean: [0.485, 0.456, 0.406]
@@ -100,11 +114,6 @@ MIX:
     name: "Mixup"
     alpha: 0.2
 
-INFERENCE:
-    name: 'ppTSM_Inference_helper'
-    num_seg: 8
-    target_size: 224
-
 PRECISEBN:
   preciseBN_interval: 5     # epoch interval to do preciseBN, default 1.
   num_iters_preciseBN: 200  # how many batches used to do preciseBN, default 200.
@@ -115,8 +124,9 @@ METRIC:
 
 INFERENCE:
     name: 'ppTSM_Inference_helper'
-    num_seg: 8
-    target_size: 224
+    num_seg: 16
+    target_size: [320, 180]
+    use_letterbox: True
 
 model_name: "ppTSM"
 log_interval: 10 #Optional, the interal of logger, default:10

paddlevideo/loader/pipelines/__init__.py

@@ -14,10 +14,10 @@
 
 from .anet_pipeline import GetMatchMap, GetVideoLabel, LoadFeat
 from .augmentations import (CenterCrop, ColorJitter, GroupRandomFlip,
-                            GroupResize, Image2Array, JitterScale, MultiCrop,
-                            Normalization, PackOutput, RandomCrop, RandomFlip,
-                            RandomResizedCrop, Scale, TenCrop, ToArray,
-                            UniformCrop, RandomGamma, MultiCenterCrop,
+                            GroupResize, Image2Array, JitterScale, Letterbox,
+                            MultiCrop, Normalization, PackOutput, RandomCrop,
+                            RandomFlip, RandomResizedCrop, Scale, TenCrop,
+                            ToArray, UniformCrop, RandomGamma, MultiCenterCrop,
                             RandomBrightness, RandomHue, RandomSaturation, YowoAug)
 from .augmentations_ava import *
 from .compose import Compose
@@ -52,5 +52,6 @@
     'RandomGamma', 'MultiCenterCrop', 'RandomBrightness', 'RandomHue',
     'RandomSaturation', 'UniformSampleFrames', 'PoseDecode', 'PoseCompact',
     'Resize', 'RandomResizedCrop_V2', 'Flip_V2', 'GeneratePoseTarget',
-    'FormatShape', 'Collect', 'RandomSaturation', 'SamplerUCF24', 'YowoAug'
+    'FormatShape', 'Collect', 'RandomSaturation', 'SamplerUCF24', 'YowoAug',
+    'Letterbox'
 ]

paddlevideo/loader/pipelines/augmentations.py

@@ -125,7 +125,9 @@ class RandomCrop(object):
     """
     Random crop images.
     Args:
-        target_size(int): Random crop a square with the target_size from an image.
+        target_size(int | tuple[int, int]): Random crop a region of this size
+            from an image. A single int crops a square; a (w, h) tuple crops
+            a rectangle of that width and height.
     """
     def __init__(self, target_size):
         self.target_size = target_size
@@ -144,11 +146,14 @@ def __call__(self, results):
             h, w = imgs.shape[2:]
         else:
             w, h = imgs[0].size
-        th, tw = self.target_size, self.target_size
+        if isinstance(self.target_size, (tuple, list)):
+            tw, th = self.target_size
+        else:
+            tw, th = self.target_size, self.target_size
 
-        assert (w >= self.target_size) and (h >= self.target_size), \
-            "image width({}) and height({}) should be larger than crop size".format(
-                w, h, self.target_size)
+        assert (w >= tw) and (h >= th), \
+            "image width({}) and height({}) should be larger than crop size({},{})".format(
+                w, h, tw, th)
 
         crop_images = []
         if 'backend' in results and results['backend'] == 'pyav':
@@ -250,7 +255,9 @@ class CenterCrop(object):
     """
     Center crop images.
     Args:
-        target_size(int): Center crop a square with the target_size from an image.
+        target_size(int | tuple[int, int]): Center crop a region of this
+            size from an image. A single int crops a square; a (w, h) tuple
+            crops a rectangle of that width and height.
         do_round(bool): Whether to round up the coordinates of the upper left corner of the cropping area. default: True
     """
     def __init__(self, target_size, do_round=True, backend='pillow'):
@@ -269,7 +276,10 @@ def __call__(self, results):
         """
         imgs = results['imgs']
         ccrop_imgs = []
-        th, tw = self.target_size, self.target_size
+        if isinstance(self.target_size, (tuple, list)):
+            tw, th = self.target_size
+        else:
+            tw, th = self.target_size, self.target_size
         if isinstance(imgs, paddle.Tensor):
             h, w = imgs.shape[-2:]
             x1 = int(round((w - tw) / 2.0)) if self.do_round else (w - tw) // 2
@@ -283,9 +293,9 @@ def __call__(self, results):
                     h, w, _ = img.shape
                 else:
                     raise NotImplementedError
-                assert (w >= self.target_size) and (h >= self.target_size), \
-                    "image width({}) and height({}) should be larger than crop size".format(
-                        w, h, self.target_size)
+                assert (w >= tw) and (h >= th), \
+                    "image width({}) and height({}) should be larger than crop size({},{})".format(
+                        w, h, tw, th)
                 x1 = int(round(
                     (w - tw) / 2.0)) if self.do_round else (w - tw) // 2
                 y1 = int(round(
@@ -298,14 +308,66 @@ def __call__(self, results):
         return results
 
 
+@PIPELINES.register()
+class Letterbox(object):
+    """
+    Resize images to fit entirely inside target_size while preserving their
+    aspect ratio, then pad the leftover space so every output has exactly
+    target_size. Unlike RandomCrop/CenterCrop, this never discards any part
+    of the frame — the whole image is always kept, at the cost of some
+    padding pixels when the source aspect ratio doesn't match target_size.
+    Args:
+        target_size(int | tuple[int, int]): (w, h) of the output canvas. A
+            single int produces a square canvas.
+        fill(int): pixel value used for the padding. Default: 0 (black).
+    """
+    def __init__(self, target_size, fill=0):
+        if isinstance(target_size, (tuple, list)):
+            self.target_w, self.target_h = target_size
+        else:
+            self.target_w, self.target_h = target_size, target_size
+        self.fill = fill
+
+    def __call__(self, results):
+        """
+        Args:
+            imgs: List where each item is a PIL.Image.
+        return:
+            letterboxed_imgs: List where each item is a PIL.Image of exactly
+                (target_w, target_h), letterboxed/pillarboxed as needed.
+        """
+        imgs = results['imgs']
+        w, h = imgs[0].size
+        scale = min(self.target_w / w, self.target_h / h)
+        new_w, new_h = int(round(w * scale)), int(round(h * scale))
+        x_off = (self.target_w - new_w) // 2
+        y_off = (self.target_h - new_h) // 2
+
+        letterboxed_imgs = []
+        for img in imgs:
+            resized = img.resize((new_w, new_h), Image.BILINEAR)
+            canvas = Image.new(img.mode, (self.target_w, self.target_h),
+                              self.fill)
+            canvas.paste(resized, (x_off, y_off))
+            letterboxed_imgs.append(canvas)
+        results['imgs'] = letterboxed_imgs
+        return results
+
+
 @PIPELINES.register()
 class MultiScaleCrop(object):
     """
     Random crop images in with multiscale sizes
     Args:
-        target_size(int): Random crop a square with the target_size from an image.
+        target_size(int | tuple[int, int]): Random crop a region of this
+            size from an image. A single int crops a square (candidate
+            scales for width/height are sampled independently, within
+            max_distort). A (w, h) tuple crops a rectangle that keeps the
+            (w, h) aspect ratio at every candidate scale, so the final
+            resize to (w, h) never distorts the image.
         scales(int): List of candidate cropping scales.
         max_distort(int): Maximum allowable deformation combination distance.
+            Ignored when target_size is a (w, h) tuple.
         fix_crop(int): Whether to fix the cutting start point.
         allow_duplication(int): Whether to allow duplicate candidate crop starting points.
         more_fix_crop(int): Whether to allow more cutting starting points.
@@ -342,31 +404,56 @@ def __call__(self, results):
         """
         imgs = results['imgs']
 
-        input_size = [self.target_size, self.target_size]
+        is_rect = isinstance(self.target_size, (tuple, list))
+        if is_rect:
+            input_size = [self.target_size[0], self.target_size[1]]
+        else:
+            input_size = [self.target_size, self.target_size]
 
         im_size = imgs[0].size
 
         # get random crop offset
         def _sample_crop_size(im_size):
             image_w, image_h = im_size[0], im_size[1]
 
-            base_size = min(image_w, image_h)
-            crop_sizes = [int(base_size * x) for x in self.scales]
-            crop_h = [
-                input_size[1] if abs(x - input_size[1]) < 3 else x
-                for x in crop_sizes
-            ]
-            crop_w = [
-                input_size[0] if abs(x - input_size[0]) < 3 else x
-                for x in crop_sizes
-            ]
+            if is_rect:
+                target_w, target_h = input_size
+                # largest (target_w:target_h)-shaped rectangle that fits
+                # inside the image, so every scaled-down candidate below
+                # keeps the exact target aspect ratio (no distortion on
+                # the final resize back to (target_w, target_h)).
+                max_w = image_h * target_w / target_h
+                if max_w <= image_w:
+                    max_h = image_h
+                else:
+                    max_w = image_w
+                    max_h = image_w * target_h / target_w
+
+                crop_pairs = []
+                for s in self.scales:
+                    w, h = int(max_w * s), int(max_h * s)
+                    if abs(w - target_w) < 3 and abs(h - target_h) < 3:
+                        w, h = target_w, target_h
+                    crop_pairs.append((w, h))
+                crop_pair = random.choice(crop_pairs)
+            else:
+                base_size = min(image_w, image_h)
+                crop_sizes = [int(base_size * x) for x in self.scales]
+                crop_h = [
+                    input_size[1] if abs(x - input_size[1]) < 3 else x
+                    for x in crop_sizes
+                ]
+                crop_w = [
+                    input_size[0] if abs(x - input_size[0]) < 3 else x
+                    for x in crop_sizes
+                ]
 
-            pairs = []
-            for i, h in enumerate(crop_h):
-                for j, w in enumerate(crop_w):
-                    if abs(i - j) <= self.max_distort:
-                        pairs.append((w, h))
-            crop_pair = random.choice(pairs)
+                pairs = []
+                for i, h in enumerate(crop_h):
+                    for j, w in enumerate(crop_w):
+                        if abs(i - j) <= self.max_distort:
+                            pairs.append((w, h))
+                crop_pair = random.choice(pairs)
             if not self.fix_crop:
                 w_offset = random.randint(0, image_w - crop_pair[0])
                 h_offset = random.randint(0, image_h - crop_pair[1])