diff --git a/.gitignore b/.gitignore index e15093f..e37bb1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ node_modules -human-service-account.json pnpm-lock.yaml +secret/** +saved/** +tflite/** diff --git a/gcp-upload.log b/gcp-upload.log deleted file mode 100644 index 4e846c9..0000000 --- a/gcp-upload.log +++ /dev/null @@ -1,103 +0,0 @@ -2022-05-30 14:41:18 INFO:  { application: '@vladmandic/human-models', version: '2.8.0' } -2022-05-30 14:41:18 INFO:  { user: 'vlado', platform: 'linux', arch: 'x64', node: 'v16.15.0' } -2022-05-30 14:41:18 DATA:  bucket metadata: { - kind: 'storage#bucket', - selfLink: 'https://www.googleapis.com/storage/v1/b/human-models', - id: 'human-models', - name: 'human-models', - projectNumber: '759769403572', - metageneration: '3', - location: 'US', - storageClass: 'STANDARD', - etag: 'CAM=', - timeCreated: '2022-05-30T11:05:24.220Z', - updated: '2022-05-30T11:13:10.381Z', - iamConfiguration: { bucketPolicyOnly: { enabled: false }, uniformBucketLevelAccess: { enabled: false }, publicAccessPrevention: 'inherited' }, - locationType: 'multi-region', - rpo: 'DEFAULT' -} -2022-05-30 14:41:18 STATE: enumerating: { folder: './models', files: 83 } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/README.md', url: 'https://storage.googleapis.com/human-models/README.md', size: '133' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/age.bin', url: 'https://storage.googleapis.com/human-models/age.bin', size: '149709' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/age.json', url: 'https://storage.googleapis.com/human-models/age.json', size: '5359' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazeface-back.bin', url: 'https://storage.googleapis.com/human-models/blazeface-back.bin', size: '334398' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazeface-back.json', url: 'https://storage.googleapis.com/human-models/blazeface-back.json', size: '4286' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazeface-front.bin', url: 'https://storage.googleapis.com/human-models/blazeface-front.bin', size: '247056' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazeface-front.json', url: 'https://storage.googleapis.com/human-models/blazeface-front.json', size: '3413' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazeface.bin', url: 'https://storage.googleapis.com/human-models/blazeface.bin', size: '334398' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazeface.json', url: 'https://storage.googleapis.com/human-models/blazeface.json', size: '4276' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazepose-detector2d.bin', url: 'https://storage.googleapis.com/human-models/blazepose-detector2d.bin', size: '7073770' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazepose-detector2d.json', url: 'https://storage.googleapis.com/human-models/blazepose-detector2d.json', size: '6977' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazepose-detector3d.bin', url: 'https://storage.googleapis.com/human-models/blazepose-detector3d.bin', size: '2816494' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazepose-detector3d.json', url: 'https://storage.googleapis.com/human-models/blazepose-detector3d.json', size: '7090' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazepose-full.bin', url: 'https://storage.googleapis.com/human-models/blazepose-full.bin', size: '5980853' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazepose-full.json', url: 'https://storage.googleapis.com/human-models/blazepose-full.json', size: '7490' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazepose-heavy.bin', url: 'https://storage.googleapis.com/human-models/blazepose-heavy.bin', size: '26168012' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazepose-heavy.json', url: 'https://storage.googleapis.com/human-models/blazepose-heavy.json', size: '13893' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazepose-lite.bin', url: 'https://storage.googleapis.com/human-models/blazepose-lite.bin', size: '2583098' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/blazepose-lite.json', url: 'https://storage.googleapis.com/human-models/blazepose-lite.json', size: '7016' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/efficientpose.bin', url: 'https://storage.googleapis.com/human-models/efficientpose.bin', size: '5228268' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/efficientpose.json', url: 'https://storage.googleapis.com/human-models/efficientpose.json', size: '6845' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/emotion.bin', url: 'https://storage.googleapis.com/human-models/emotion.bin', size: '763176' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/emotion.json', url: 'https://storage.googleapis.com/human-models/emotion.json', size: '1847' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/faceboxes.bin', url: 'https://storage.googleapis.com/human-models/faceboxes.bin', size: '1868336' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/faceboxes.json', url: 'https://storage.googleapis.com/human-models/faceboxes.json', size: '14400' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/facemesh-attention-alt.bin', url: 'https://storage.googleapis.com/human-models/facemesh-attention-alt.bin', size: '2212974' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/facemesh-attention-alt.json', url: 'https://storage.googleapis.com/human-models/facemesh-attention-alt.json', size: '19896' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/facemesh-attention.bin', url: 'https://storage.googleapis.com/human-models/facemesh-attention.bin', size: '2198535' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/facemesh-attention.json', url: 'https://storage.googleapis.com/human-models/facemesh-attention.json', size: '35457' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/facemesh-detection-full.bin', url: 'https://storage.googleapis.com/human-models/facemesh-detection-full.bin', size: '546170' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/facemesh-detection-full.json', url: 'https://storage.googleapis.com/human-models/facemesh-detection-full.json', size: '8868' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/facemesh-detection-short.bin', url: 'https://storage.googleapis.com/human-models/facemesh-detection-short.bin', size: '189108' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/facemesh-detection-short.json', url: 'https://storage.googleapis.com/human-models/facemesh-detection-short.json', size: '4451' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/facemesh-orig.bin', url: 'https://storage.googleapis.com/human-models/facemesh-orig.bin', size: '2735233' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/facemesh-orig.json', url: 'https://storage.googleapis.com/human-models/facemesh-orig.json', size: '4635' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/facemesh.bin', url: 'https://storage.googleapis.com/human-models/facemesh.bin', size: '1358557' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/facemesh.json', url: 'https://storage.googleapis.com/human-models/facemesh.json', size: '5005' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/faceres-deep.bin', url: 'https://storage.googleapis.com/human-models/faceres-deep.bin', size: '13008320' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/faceres-deep.json', url: 'https://storage.googleapis.com/human-models/faceres-deep.json', size: '4316' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/faceres.bin', url: 'https://storage.googleapis.com/human-models/faceres.bin', size: '6362060' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/faceres.json', url: 'https://storage.googleapis.com/human-models/faceres.json', size: '4415' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/gear.bin', url: 'https://storage.googleapis.com/human-models/gear.bin', size: '1378785' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/gear.json', url: 'https://storage.googleapis.com/human-models/gear.json', size: '2189' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/gender-ssrnet-imdb.bin', url: 'https://storage.googleapis.com/human-models/gender-ssrnet-imdb.bin', size: '150079' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/gender-ssrnet-imdb.json', url: 'https://storage.googleapis.com/human-models/gender-ssrnet-imdb.json', size: '5332' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/gender.bin', url: 'https://storage.googleapis.com/human-models/gender.bin', size: '188463' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/gender.json', url: 'https://storage.googleapis.com/human-models/gender.json', size: '2398' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/handdetect.bin', url: 'https://storage.googleapis.com/human-models/handdetect.bin', size: '3280598' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/handdetect.json', url: 'https://storage.googleapis.com/human-models/handdetect.json', size: '6314' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/handlandmark-full.bin', url: 'https://storage.googleapis.com/human-models/handlandmark-full.bin', size: '3750533' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/handlandmark-full.json', url: 'https://storage.googleapis.com/human-models/handlandmark-full.json', size: '5207' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/handlandmark-lite.bin', url: 'https://storage.googleapis.com/human-models/handlandmark-lite.bin', size: '1760908' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/handlandmark-lite.json', url: 'https://storage.googleapis.com/human-models/handlandmark-lite.json', size: '5233' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/handlandmark-sparse.bin', url: 'https://storage.googleapis.com/human-models/handlandmark-sparse.bin', size: '998534' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/handlandmark-sparse.json', url: 'https://storage.googleapis.com/human-models/handlandmark-sparse.json', size: '5577' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/handskeleton.bin', url: 'https://storage.googleapis.com/human-models/handskeleton.bin', size: '5140792' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/handskeleton.json', url: 'https://storage.googleapis.com/human-models/handskeleton.json', size: '5681' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/handtrack.bin', url: 'https://storage.googleapis.com/human-models/handtrack.bin', size: '2299744' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/handtrack.json', url: 'https://storage.googleapis.com/human-models/handtrack.json', size: '25694' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/iris.bin', url: 'https://storage.googleapis.com/human-models/iris.bin', size: '2430503' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/iris.json', url: 'https://storage.googleapis.com/human-models/iris.json', size: '6171' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/liveness.bin', url: 'https://storage.googleapis.com/human-models/liveness.bin', size: '547713' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/liveness.json', url: 'https://storage.googleapis.com/human-models/liveness.json', size: '1610' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/mb3-centernet.bin', url: 'https://storage.googleapis.com/human-models/mb3-centernet.bin', size: '3134746' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/mb3-centernet.json', url: 'https://storage.googleapis.com/human-models/mb3-centernet.json', size: '10096' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/meet.bin', url: 'https://storage.googleapis.com/human-models/meet.bin', size: '348561' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/meet.json', url: 'https://storage.googleapis.com/human-models/meet.json', size: '6457' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/mobileface.bin', url: 'https://storage.googleapis.com/human-models/mobileface.bin', size: '2031790' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/mobileface.json', url: 'https://storage.googleapis.com/human-models/mobileface.json', size: '2887' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/mobilefacenet.bin', url: 'https://storage.googleapis.com/human-models/mobilefacenet.bin', size: '4825093' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/mobilefacenet.json', url: 'https://storage.googleapis.com/human-models/mobilefacenet.json', size: '5671' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/movenet-lightning.bin', url: 'https://storage.googleapis.com/human-models/movenet-lightning.bin', size: '4318369' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/movenet-lightning.json', url: 'https://storage.googleapis.com/human-models/movenet-lightning.json', size: '7578' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/movenet-multipose.bin', url: 'https://storage.googleapis.com/human-models/movenet-multipose.bin', size: '8785221' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/movenet-multipose.json', url: 'https://storage.googleapis.com/human-models/movenet-multipose.json', size: '12084' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/movenet-thunder.bin', url: 'https://storage.googleapis.com/human-models/movenet-thunder.bin', size: '11563091' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/movenet-thunder.json', url: 'https://storage.googleapis.com/human-models/movenet-thunder.json', size: '7569' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/nanodet.bin', url: 'https://storage.googleapis.com/human-models/nanodet.bin', size: '7015081' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/nanodet.json', url: 'https://storage.googleapis.com/human-models/nanodet.json', size: '7297' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/posenet.bin', url: 'https://storage.googleapis.com/human-models/posenet.bin', size: '4692328' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/posenet.json', url: 'https://storage.googleapis.com/human-models/posenet.json', size: '2414' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/selfie.bin', url: 'https://storage.googleapis.com/human-models/selfie.bin', size: '201212' } -2022-05-30 14:41:18 DATA:  exists: { file: 'models/selfie.json', url: 'https://storage.googleapis.com/human-models/selfie.json', size: '5876' } -2022-05-30 14:41:18 INFO:  bucket info: { files: 83, size: 151310072 } diff --git a/package.json b/package.json index 875ef78..1ca67ad 100644 --- a/package.json +++ b/package.json @@ -22,10 +22,10 @@ "tfjs-models" ], "dependencies": { - "@google-cloud/storage": "^6.2.3", - "@vladmandic/human": "^2.8.1", - "@vladmandic/pilogger": "^0.4.5", - "eslint": "^8.20.0", + "@google-cloud/storage": "^6.4.1", + "@vladmandic/human": "^2.9.3", + "@vladmandic/pilogger": "^0.4.6", + "eslint": "^8.22.0", "eslint-config-airbnb-base": "^15.0.0", "eslint-plugin-import": "^2.26.0", "eslint-plugin-node": "^11.1.0" diff --git a/gcp-upload.js b/src/gcp-upload.js similarity index 94% rename from gcp-upload.js rename to src/gcp-upload.js index 0a10925..34125e0 100644 --- a/gcp-upload.js +++ b/src/gcp-upload.js @@ -1,6 +1,5 @@ /** * This is internal-only script that uploads all models to Google Cloud Storage bucket - * */ /* GCP bucket auth init @@ -13,11 +12,11 @@ const fs = require('fs'); const path = require('path'); const log = require('@vladmandic/pilogger'); const { Storage } = require('@google-cloud/storage'); -const authJson = require('./human-service-account.json'); +const authJson = require('../secret/human-service-account.json'); -const localDir = './models'; -const keyFilename = './human-service-account.json'; +const keyFilename = 'secret/human-service-account.json'; const bucketName = 'human-models'; +const localDir = 'models'; async function main() { log.headerJson(); diff --git a/src/tfjs-to-saved.py b/src/tfjs-to-saved.py new file mode 100644 index 0000000..ee80931 --- /dev/null +++ b/src/tfjs-to-saved.py @@ -0,0 +1,36 @@ +import os +import sys +import glob +import tensorflow as tf +import tfjs_graph_converter.api as tfjs + +graphDir = 'models/' +savedDir = 'saved/' +tfliteDir = 'tflite/' + +def main() -> None: + for f in glob.glob(os.path.join(graphDir, '*.json')): + modelName = os.path.basename(f).split('.')[0] + print('graph model: ' + modelName + ' path: ' + f) + savedModel = os.path.join(savedDir, modelName) + try: + tfjs.graph_model_to_saved_model(f, savedModel) # type: ignore + except: + print('saved convert failed') + else: + converter = tf.lite.TFLiteConverter.from_saved_model(savedModel) + converter.optimizations = [tf.lite.Optimize.DEFAULT] + converter.target_spec.supported_ops = [ tf.lite.OpsSet.TFLITE_BUILTINS, tf.lite.OpsSet.SELECT_TF_OPS ] + converter.target_spec.supported_types = [tf.float16] + tfliteModel = os.path.join(tfliteDir, modelName) + try: + tflite_model = converter.convert() + except: + print('tflite convert failed') + else: + with open(tfliteModel, 'wb') as f: + f.write(tflite_model) + print('saved:' + savedModel + ' tflite: ' + tfliteModel) + +if __name__ == '__main__': + main()