MH
1
Mike Heavers 6 years ago
Toggle between webcam and still images here.
MH
2
Mike Heavers 6 years ago
This is the first layer of the network. Here, we are analyzing the pixels direct
MH
3
Mike Heavers 6 years ago
In pooling layers, we reduce the dimensionality of our network, getting averages
MH
4
Mike Heavers 6 years ago
Each further convolution makes assumptions not upon the original images, but upo
MH
5
Mike Heavers 6 years ago
We can view the filters being used in the CNN here.
MH
6
Mike Heavers 6 years ago
Local response normalization refers to the ability of an excited neuron to subdu
MH
7
Mike Heavers 6 years ago
Fully connected layer - where the high level reasoning is done. Tunes the weight
MH
Mike Heavers 6 years ago
need to link to google drive
MH
Mike Heavers 6 years ago (edited)
TODO: Add examples
- 01-Visual Machine Learning
- OVERVIEW
- Image Classification
- Image Regression
- Style Transfer
- Current State of Visual Machine Learning
- MACHINE LEARNING WITH IMAGES
- CNNs
- Pooling
- Feature Extraction
- How Neural Networks Understand Images
- Convnet Viewer
- Looking at Convolutions
- Synthesizing Images to Identify Specific Categories
- Deep Dream
- Why so many puppy slugs?
- Image Classification
- When might this be used?
- Using a pre-built image classifier
- Training our first model
- Video Classifier
- Pre-trained webcam classifier
- Image Regressor
- Style Transfer
- What is a style?
- Why is ML so processor intensive?
- Style Transfer Examples
- Basic Style Transfer
- Applying one style to many images
- Applying a style to a sequence of images / animated GIF
- Style Transfer for Video
- Saving the Results
- Creating your own style
- Process
- Tuning the Results
- Style Transfer References
- Current Examples of Machine Learning for Images/Video
- SFV - Skills from Videos
- CycleGAN
- Style Transfer For-Ev-Er
- FOR NEXT CLASS
OVERVIEW
MACHINE LEARNING WITH IMAGES
CNNs
How Neural Networks Understand Images
Convnet Viewer
Image Classification
let featureExtractor; //the thing that allows to make assumptions about a specific group
let classifier; //the thing that separates those assumptions into one group vs another
let video; //the webcam image
let loss; //when comparing training vs. validation, how well is the model doing per iteration of optimization
let pose2Images = 0;
let pose1Images = 0;
function setup() {
noCanvas(); //we don't actually want to display anything other than webcam on the screen
video = createCapture(VIDEO); //Create a video element
video.parent('videoContainer'); // Append it to the videoContainer DOM element
featureExtractor = ml5.featureExtractor('MobileNet', modelReady); // utilize MobileNet's abilities to extract features from images
classifier = featureExtractor.classification(video, videoReady); // Create a new classifier to identify those features and give it our video to analyze
setupButtons(); // Create the UI for our user to click on to make things happen
}
function setupButtons() {
// When the pose1 button is pressed, add the current frame as pose 1
buttonA = select('#pose1Button');
buttonA.mousePressed(function() {
classifier.addImage('pose1');
select('#numPose1Images').html(pose1Images++);
});
// When the pose2 button is pressed, assign the current frame as pose 2
buttonB = select('#pose2Button');
buttonB.mousePressed(function() {
classifier.addImage('pose2');
select('#numPose2Images').html(pose2Images++);
});
// Train Button
train = select('#train');
train.mousePressed(function() {
classifier.train(function(lossValue) {
if (lossValue) { //what is the term for what is happening here?
loss = lossValue;
select('#loss').html('Loss: ' + loss);
} else { //we are at the bottom of our descent
select('#loss').html('Done Training! Final Loss: ' + loss);
}
});
});
// Predict Button
buttonPredict = select('#buttonPredict');
buttonPredict.mousePressed(classify);
}
function classify() {
classifier.classify(gotResults);
}
// Show the results
function gotResults(err, result) {
if (err) {
console.error(err);
}
select('#result').html(result);
classify();
}
ml5.imageClassifier('MobileNet', video)
.then((classifier) => {
loading.innerText = "";
video.play();
loop(classifier);
});
const loop = (classifier) => {
classifier.predict()
.then(results => {
result.innerText = results[0].className;
probability.innerText = results[0].probability.toFixed(4);
loop(classifier) // Call again to create a loop
})
}
let featureExtractor;
let regressor;
let video;
let loss;
let slider;
let samples = 0;
let positionX = 140;
function setup() {
createCanvas(340, 280);
// Create a video element
video = createCapture(VIDEO);
// Append it to the videoContainer DOM element
video.hide();
// Extract the features from MobileNet
featureExtractor = ml5.featureExtractor('MobileNet', modelReady);
// Create a new regressor using those features and give the video we want to use
regressor = featureExtractor.regression(video, videoReady);
// Create the UI buttons
setupButtons();
}
// A util function to create UI buttons
function setupButtons() {
slider = select('#slider');
select('#addSample').mousePressed(function() {
regressor.addImage(slider.value());
select('#amountOfSamples').html(samples++);
});
// Train Button
select('#train').mousePressed(function() {
regressor.train(function(lossValue) {
if (lossValue) {
loss = lossValue;
select('#loss').html('Loss: ' + loss);
} else {
select('#loss').html('Done Training! Final Loss: ' + loss);
}
});
});
// Predict Button
select('#buttonPredict').mousePressed(predict);
}
// A function to be called when the model has been loaded
function modelReady() {
select('#modelStatus').html('Model loaded!');
}
// A function to be called when the video has loaded
function videoReady() {
select('#videoStatus').html('Video ready!');
}
// Classify the current frame.
function predict() {
regressor.predict(gotResults);
}
// Show the results
function gotResults(err, result) {
if (err) {
console.error(err);
}
positionX = map(result, 0, 1, 0, width);
slider.value(result);
predict();
}
function draw() {
image(video, 0, 0, 340, 280);
noStroke();
fill(255, 0, 0);
rect(positionX, 120, 50, 50);
}
Style Transfer
Style Transfer Examples
<p id="statusMsg">Loading Models...</p>
<button id="transferBtn">Transfer!</button>
<img src="img/patagonia.jpg" alt="input img" id="inputImg">
<div id="styleA">
<p>Style A: <a href="https://en.wikipedia.org/wiki/The_Great_Wave_off_Kanagawa">The Great Wave off Kanagawa, 1829 - Katsushika Hokusai</a></p>
<img src="img/wave.jpg" alt="style one">
</div>
let inputImg;
let statusMsg;
let transferBtn;
let style1;
let style2;
let style3;
inputImg = select('#inputImg');
style1 = ml5.styleTransfer('models/wave', modelLoaded);
function modelLoaded() {
// Check if both models are loaded
if(style1.ready && style2.ready && style3.ready){
statusMsg.html('Ready!')
}
}
style1.transfer(inputImg, function(err, result) {
createImg(result.src).parent('styleA');
});
for (let i=0;i<numImages;i++){
style1.transfer(inputImgArr[i], function(err, result) {
if (result){
createImg(result.src).parent('output-images');
}
});
}
createCanvas(320, 240).parent('canvasContainer');
video = createVideo('/assets/videos/house.mov');
select('#startStop').mousePressed(startStop);
function startStop() {
if (isTransferring) {
select('#startStop').html('Start');
video.pause();
} else {
select('#startStop').html('Stop');
video.loop();
// Make a transfer using the video
style.transfer(gotResult);
}
isTransferring = !isTransferring;
}
function gotResult(err, img) {
resultImg.attribute('src', img.src);
if (isTransferring) {
style.transfer(gotResult);
}
}
function draw(){
// Switch between showing the raw camera or the style
if (isTransferring) {
image(resultImg, 0, 0, 320, 240);
} else {
image(video, 0, 0, 320, 240);
}
}
saveCanvas(c, `kinect${i}`, 'jpg');
Creating your own style
npm install -g paperspace-node
paperspace login
paperspace jobs create --container cvalenzuelab/styletransfer --machineType P5000 --command './run.sh' --project 'Style Transfer training'
Current Examples of Machine Learning for Images/Video
FOR NEXT CLASS