Xtykutl

I'm currently attempting to figure out how to convert a input png into a tensor with tensorflow.js so I can feed it into my model for training. Currently I'm capturing the image, saving it locally, reading it with fs.readFileSync, and then creating a buffer. Where i'm a bit lost is normalizing the buffer values from 0-244 to 0-1, then creating a tensor from this buffer to feed into the model.fit function as the X arg. I also don't really know how to set up my labels file and properly convert that into a buffer for the Y arg. (https://js.tensorflow.org/api/0.11.2/#tf.Model.fit) Any insight into the proper usage / configuration of images into tensors for using tensorflow.js would be greatly appreciated.

Repo is here;
https://github.com/Durban-Designer/Fighter-Ai

code for loading local image in data.js;

const tf = require('@tensorflow/tfjs');

const assert = require('assert');

const IMAGE_HEADER_BYTES = 32;

const IMAGE_HEIGHT = 600;

const IMAGE_WIDTH = 800;

const IMAGE_FLAT_SIZE = IMAGE_HEIGHT * IMAGE_WIDTH;



function loadHeaderValues(buffer, headerLength) {

  const headerValues = ;

  for (let i = 0; i < headerLength / 4; i++) {

    headerValues[i] = buffer.readUInt32BE(i * 4);

  }

  return headerValues;

}



...

...

class Dataset {

 async loadLocalImage(filename) {

 const buffer = fs.readFileSync(filename);



 const headerBytes = IMAGE_HEADER_BYTES;

 const recordBytes = IMAGE_HEIGHT * IMAGE_WIDTH;



 const headerValues = loadHeaderValues(buffer, headerBytes);

 console.log(headerValues, buffer);

 assert.equal(headerValues[5], IMAGE_HEIGHT);

 assert.equal(headerValues[4], IMAGE_WIDTH);



 const images = ;

 let index = headerBytes;

 while (index < buffer.byteLength) {

  const array = new Float32Array(recordBytes);

  for (let i = 0; i < recordBytes; i++) {

    // Normalize the pixel values into the 0-1 interval, from

    // the original 0-255 interval.

    array[i] = buffer.readUInt8(index++) / 255;

  }

  images.push(array);

 }



 assert.equal(images.length, headerValues[1]);

 return images;

 }

}

module.exports = new Dataset();

image capture loop in app.js;

const ioHook = require("iohook");

const tf = require('@tensorflow/tfjs');

var screenCap = require('desktop-screenshot');

require('@tensorflow/tfjs-node');

const data = require('./src/data');

const virtKeys = require('./src/virtKeys');

const model = require('./src/model');

var dir = __dirname;

var paused = true;

var loopInterval,

  image,

  imageData,

  result



ioHook.on('keyup', event => {

  if (event.keycode === 88) {

    if (paused) {

      paused = false;

      gameLoop();

    } else {

      paused = true;

    }

  }

});



ioHook.start();

function gameLoop () {

  if (!paused) {

    screenCap(dir + '\image.png', {width: 800, height: 600, quality: 60}, function (error, complete) {

      if (error) {

        console.log(error);

      } else {

        imageData = await data.getImage(dir + '\image.png')

        console.log(imageData);

        result = model.predict(imageData, {batchSize: 4});

        console.log(result);

        gameLoop();

      }

    })

  }

}

I know I use model.predict here, I wanted to get the actual image to tensor part working then figure out labels and model.fit() in train-tensor.js in the repo. I don't have any actual working code for training so I didn't include it in this question, sorry if it caused any confusion.

Thank you again!

Edit final working code

const { Image, createCanvas } = require('canvas');

const canvas = createCanvas(800, 600);

const ctx = canvas.getContext('2d');



async function loadLocalImage (filename) {

  try {

    var img = new Image()

    img.onload = () => ctx.drawImage(img, 0, 0);

    img.onerror = err => { throw err };

    img.src = filename;

    image = tf.fromPixels(canvas);

    return image;

  } catch (err) {

    console.log(err);

  }

}

...

...

async getImage(filename) {

    try {

      this.image = await loadLocalImage(filename);

    } catch (error) {

      console.log('error loading image', error);

    }

    return this.image;

  }

tensorflowjs already has a method for this: tf.fromPixels().

You just need to load the image into on of the accepted types(ImageData|HTMLImageElement|HTMLCanvasElement|HTMLVideoElement).

Your image loading Promise returns nothing because your async function doesn't return anything, just your callback, to fix this you need to create and resolve a promise yourself:

const imageGet = require('get-image-data');

async loadLocalImage(filename) {

    return new Promise((res,rej)=>{

    imageGet(filename, (err, info) => {

      if(err){

         rej(err);

         return;

      }

      const image = tf.fromPixels(info.data)

      console.log(image, '127');

      res(image);

    });

  }