| from .intern_action import intern_action_b16 |
| from huggingface_hub import hf_hub_download |
| |
| import torch |
| import torch.nn as nn |
| import torchvision.transforms as T |
| import torch.nn.functional as F |
| import numpy as np |
|
|
| from .processing import ( |
| GroupNormalize, GroupScale, GroupCenterCrop, |
| Stack, ToTorchFormatTensor |
| ) |
|
|
| class Intern_Action(nn.Module): |
| def __init__(self, model): |
| super().__init__() |
| self.backbone = model |
| |
| def forward(self, x): |
| return self.backbone(x) |
|
|
| def get_index(num_frames, num_segments=8): |
| seg_size = float(num_frames - 1) / num_segments |
| start = int(seg_size / 2) |
| offsets = np.array([ |
| start + int(np.round(seg_size * idx)) for idx in range(num_segments) |
| ]) |
| return offsets |
|
|
| def transform_action(): |
| |
| crop_size = 224 |
| scale_size = 256 |
| input_mean = [0.485, 0.456, 0.406] |
| input_std = [0.229, 0.224, 0.225] |
|
|
| return T.Compose([ |
| |
| GroupScale(int(scale_size)), |
| GroupCenterCrop(crop_size), |
| Stack(), |
| ToTorchFormatTensor(), |
| GroupNormalize(input_mean, input_std) |
| ]) |
|
|
| def load_intern_action(device): |
| |
| kinetics_id_to_classname = {} |
| for k, v in kinetics_classnames.items(): |
| kinetics_id_to_classname[k] = v |
|
|
| model_path = hf_hub_download(repo_id="Andy1621/uniformerv2", filename="k400+k710_uniformerv2_b16_8x224.pyth") |
| |
| model = Intern_Action(intern_action_b16(pretrained=False, t_size=8, no_lmhra=True, temporal_downsample=False)) |
| state_dict = torch.load(model_path, map_location=device) |
| model.load_state_dict(state_dict) |
| |
| model = model.to(device) |
| model = model.eval() |
| return model |
|
|
| def cut_frame_to_8(data): |
| index = np.linspace(0, len(data)-1, 8).astype(int) |
| return data[index] |
|
|
| kinetics_classnames = { |
| "0": "riding a bike", |
| "1": "marching", |
| "2": "dodgeball", |
| "3": "playing cymbals", |
| "4": "checking tires", |
| "5": "roller skating", |
| "6": "tasting beer", |
| "7": "clapping", |
| "8": "drawing", |
| "9": "juggling fire", |
| "10": "bobsledding", |
| "11": "petting animal (not cat)", |
| "12": "spray painting", |
| "13": "training dog", |
| "14": "eating watermelon", |
| "15": "building cabinet", |
| "16": "applauding", |
| "17": "playing harp", |
| "18": "balloon blowing", |
| "19": "sled dog racing", |
| "20": "wrestling", |
| "21": "pole vault", |
| "22": "hurling (sport)", |
| "23": "riding scooter", |
| "24": "shearing sheep", |
| "25": "sweeping floor", |
| "26": "eating carrots", |
| "27": "skateboarding", |
| "28": "dunking basketball", |
| "29": "disc golfing", |
| "30": "eating spaghetti", |
| "31": "playing flute", |
| "32": "riding mechanical bull", |
| "33": "making sushi", |
| "34": "trapezing", |
| "35": "picking fruit", |
| "36": "stretching leg", |
| "37": "playing ukulele", |
| "38": "tying tie", |
| "39": "skydiving", |
| "40": "playing cello", |
| "41": "jumping into pool", |
| "42": "shooting goal (soccer)", |
| "43": "trimming trees", |
| "44": "bookbinding", |
| "45": "ski jumping", |
| "46": "walking the dog", |
| "47": "riding unicycle", |
| "48": "shaving head", |
| "49": "hopscotch", |
| "50": "playing piano", |
| "51": "parasailing", |
| "52": "bartending", |
| "53": "kicking field goal", |
| "54": "finger snapping", |
| "55": "dining", |
| "56": "yawning", |
| "57": "peeling potatoes", |
| "58": "canoeing or kayaking", |
| "59": "front raises", |
| "60": "laughing", |
| "61": "dancing macarena", |
| "62": "digging", |
| "63": "reading newspaper", |
| "64": "hitting baseball", |
| "65": "clay pottery making", |
| "66": "exercising with an exercise ball", |
| "67": "playing saxophone", |
| "68": "shooting basketball", |
| "69": "washing hair", |
| "70": "lunge", |
| "71": "brushing hair", |
| "72": "curling hair", |
| "73": "kitesurfing", |
| "74": "tapping guitar", |
| "75": "bending back", |
| "76": "skipping rope", |
| "77": "situp", |
| "78": "folding paper", |
| "79": "cracking neck", |
| "80": "assembling computer", |
| "81": "cleaning gutters", |
| "82": "blowing out candles", |
| "83": "shaking hands", |
| "84": "dancing gangnam style", |
| "85": "windsurfing", |
| "86": "tap dancing", |
| "87": "skiing (not slalom or crosscountry)", |
| "88": "bandaging", |
| "89": "push up", |
| "90": "doing nails", |
| "91": "punching person (boxing)", |
| "92": "bouncing on trampoline", |
| "93": "scrambling eggs", |
| "94": "singing", |
| "95": "cleaning floor", |
| "96": "krumping", |
| "97": "drumming fingers", |
| "98": "snowmobiling", |
| "99": "gymnastics tumbling", |
| "100": "headbanging", |
| "101": "catching or throwing frisbee", |
| "102": "riding elephant", |
| "103": "bee keeping", |
| "104": "feeding birds", |
| "105": "snatch weight lifting", |
| "106": "mowing lawn", |
| "107": "fixing hair", |
| "108": "playing trumpet", |
| "109": "flying kite", |
| "110": "crossing river", |
| "111": "swinging legs", |
| "112": "sanding floor", |
| "113": "belly dancing", |
| "114": "sneezing", |
| "115": "clean and jerk", |
| "116": "side kick", |
| "117": "filling eyebrows", |
| "118": "shuffling cards", |
| "119": "recording music", |
| "120": "cartwheeling", |
| "121": "feeding fish", |
| "122": "folding clothes", |
| "123": "water skiing", |
| "124": "tobogganing", |
| "125": "blowing leaves", |
| "126": "smoking", |
| "127": "unboxing", |
| "128": "tai chi", |
| "129": "waxing legs", |
| "130": "riding camel", |
| "131": "slapping", |
| "132": "tossing salad", |
| "133": "capoeira", |
| "134": "playing cards", |
| "135": "playing organ", |
| "136": "playing violin", |
| "137": "playing drums", |
| "138": "tapping pen", |
| "139": "vault", |
| "140": "shoveling snow", |
| "141": "playing tennis", |
| "142": "getting a tattoo", |
| "143": "making a sandwich", |
| "144": "making tea", |
| "145": "grinding meat", |
| "146": "squat", |
| "147": "eating doughnuts", |
| "148": "ice fishing", |
| "149": "snowkiting", |
| "150": "kicking soccer ball", |
| "151": "playing controller", |
| "152": "giving or receiving award", |
| "153": "welding", |
| "154": "throwing discus", |
| "155": "throwing axe", |
| "156": "ripping paper", |
| "157": "swimming butterfly stroke", |
| "158": "air drumming", |
| "159": "blowing nose", |
| "160": "hockey stop", |
| "161": "taking a shower", |
| "162": "bench pressing", |
| "163": "planting trees", |
| "164": "pumping fist", |
| "165": "climbing tree", |
| "166": "tickling", |
| "167": "high kick", |
| "168": "waiting in line", |
| "169": "slacklining", |
| "170": "tango dancing", |
| "171": "hurdling", |
| "172": "carrying baby", |
| "173": "celebrating", |
| "174": "sharpening knives", |
| "175": "passing American football (in game)", |
| "176": "headbutting", |
| "177": "playing recorder", |
| "178": "brush painting", |
| "179": "garbage collecting", |
| "180": "robot dancing", |
| "181": "shredding paper", |
| "182": "pumping gas", |
| "183": "rock climbing", |
| "184": "hula hooping", |
| "185": "braiding hair", |
| "186": "opening present", |
| "187": "texting", |
| "188": "decorating the christmas tree", |
| "189": "answering questions", |
| "190": "playing keyboard", |
| "191": "writing", |
| "192": "bungee jumping", |
| "193": "sniffing", |
| "194": "eating burger", |
| "195": "playing accordion", |
| "196": "making pizza", |
| "197": "playing volleyball", |
| "198": "tasting food", |
| "199": "pushing cart", |
| "200": "spinning poi", |
| "201": "cleaning windows", |
| "202": "arm wrestling", |
| "203": "changing oil", |
| "204": "swimming breast stroke", |
| "205": "tossing coin", |
| "206": "deadlifting", |
| "207": "hoverboarding", |
| "208": "cutting watermelon", |
| "209": "cheerleading", |
| "210": "snorkeling", |
| "211": "washing hands", |
| "212": "eating cake", |
| "213": "pull ups", |
| "214": "surfing water", |
| "215": "eating hotdog", |
| "216": "holding snake", |
| "217": "playing harmonica", |
| "218": "ironing", |
| "219": "cutting nails", |
| "220": "golf chipping", |
| "221": "shot put", |
| "222": "hugging", |
| "223": "playing clarinet", |
| "224": "faceplanting", |
| "225": "trimming or shaving beard", |
| "226": "drinking shots", |
| "227": "riding mountain bike", |
| "228": "tying bow tie", |
| "229": "swinging on something", |
| "230": "skiing crosscountry", |
| "231": "unloading truck", |
| "232": "cleaning pool", |
| "233": "jogging", |
| "234": "ice climbing", |
| "235": "mopping floor", |
| "236": "making bed", |
| "237": "diving cliff", |
| "238": "washing dishes", |
| "239": "grooming dog", |
| "240": "weaving basket", |
| "241": "frying vegetables", |
| "242": "stomping grapes", |
| "243": "moving furniture", |
| "244": "cooking sausages", |
| "245": "doing laundry", |
| "246": "dying hair", |
| "247": "knitting", |
| "248": "reading book", |
| "249": "baby waking up", |
| "250": "punching bag", |
| "251": "surfing crowd", |
| "252": "cooking chicken", |
| "253": "pushing car", |
| "254": "springboard diving", |
| "255": "swing dancing", |
| "256": "massaging legs", |
| "257": "beatboxing", |
| "258": "breading or breadcrumbing", |
| "259": "somersaulting", |
| "260": "brushing teeth", |
| "261": "stretching arm", |
| "262": "juggling balls", |
| "263": "massaging person's head", |
| "264": "eating ice cream", |
| "265": "extinguishing fire", |
| "266": "hammer throw", |
| "267": "whistling", |
| "268": "crawling baby", |
| "269": "using remote controller (not gaming)", |
| "270": "playing cricket", |
| "271": "opening bottle", |
| "272": "playing xylophone", |
| "273": "motorcycling", |
| "274": "driving car", |
| "275": "exercising arm", |
| "276": "passing American football (not in game)", |
| "277": "playing kickball", |
| "278": "sticking tongue out", |
| "279": "flipping pancake", |
| "280": "catching fish", |
| "281": "eating chips", |
| "282": "shaking head", |
| "283": "sword fighting", |
| "284": "playing poker", |
| "285": "cooking on campfire", |
| "286": "doing aerobics", |
| "287": "paragliding", |
| "288": "using segway", |
| "289": "folding napkins", |
| "290": "playing bagpipes", |
| "291": "gargling", |
| "292": "skiing slalom", |
| "293": "strumming guitar", |
| "294": "javelin throw", |
| "295": "waxing back", |
| "296": "riding or walking with horse", |
| "297": "plastering", |
| "298": "long jump", |
| "299": "parkour", |
| "300": "wrapping present", |
| "301": "egg hunting", |
| "302": "archery", |
| "303": "cleaning toilet", |
| "304": "swimming backstroke", |
| "305": "snowboarding", |
| "306": "catching or throwing baseball", |
| "307": "massaging back", |
| "308": "blowing glass", |
| "309": "playing guitar", |
| "310": "playing chess", |
| "311": "golf driving", |
| "312": "presenting weather forecast", |
| "313": "rock scissors paper", |
| "314": "high jump", |
| "315": "baking cookies", |
| "316": "using computer", |
| "317": "washing feet", |
| "318": "arranging flowers", |
| "319": "playing bass guitar", |
| "320": "spraying", |
| "321": "cutting pineapple", |
| "322": "waxing chest", |
| "323": "auctioning", |
| "324": "jetskiing", |
| "325": "drinking", |
| "326": "busking", |
| "327": "playing monopoly", |
| "328": "salsa dancing", |
| "329": "waxing eyebrows", |
| "330": "watering plants", |
| "331": "zumba", |
| "332": "chopping wood", |
| "333": "pushing wheelchair", |
| "334": "carving pumpkin", |
| "335": "building shed", |
| "336": "making jewelry", |
| "337": "catching or throwing softball", |
| "338": "bending metal", |
| "339": "ice skating", |
| "340": "dancing charleston", |
| "341": "abseiling", |
| "342": "climbing a rope", |
| "343": "crying", |
| "344": "cleaning shoes", |
| "345": "dancing ballet", |
| "346": "driving tractor", |
| "347": "triple jump", |
| "348": "throwing ball", |
| "349": "getting a haircut", |
| "350": "running on treadmill", |
| "351": "climbing ladder", |
| "352": "blasting sand", |
| "353": "playing trombone", |
| "354": "drop kicking", |
| "355": "country line dancing", |
| "356": "changing wheel", |
| "357": "feeding goats", |
| "358": "tying knot (not on a tie)", |
| "359": "setting table", |
| "360": "shaving legs", |
| "361": "kissing", |
| "362": "riding mule", |
| "363": "counting money", |
| "364": "laying bricks", |
| "365": "barbequing", |
| "366": "news anchoring", |
| "367": "smoking hookah", |
| "368": "cooking egg", |
| "369": "peeling apples", |
| "370": "yoga", |
| "371": "sharpening pencil", |
| "372": "dribbling basketball", |
| "373": "petting cat", |
| "374": "playing ice hockey", |
| "375": "milking cow", |
| "376": "shining shoes", |
| "377": "juggling soccer ball", |
| "378": "scuba diving", |
| "379": "playing squash or racquetball", |
| "380": "drinking beer", |
| "381": "sign language interpreting", |
| "382": "playing basketball", |
| "383": "breakdancing", |
| "384": "testifying", |
| "385": "making snowman", |
| "386": "golf putting", |
| "387": "playing didgeridoo", |
| "388": "biking through snow", |
| "389": "sailing", |
| "390": "jumpstyle dancing", |
| "391": "water sliding", |
| "392": "grooming horse", |
| "393": "massaging feet", |
| "394": "playing paintball", |
| "395": "making a cake", |
| "396": "bowling", |
| "397": "contact juggling", |
| "398": "applying cream", |
| "399": "playing badminton" |
| } |
|
|
|
|
