I was working on my app, and was trying to figure out how to get my model to output a result in my react native app.
In the app there is a page which prompts the user to take a photo, and from that image the model is supposed to print out the result on the next page after the user presses the button. I was having difficulty getting the model to output the result as I don't know where the image that the user had taken goes and how to use it. I was also have trouble finding out how to print the result in a certain font and text size as whenever I had tried to print the result it would print the command.
code to take the image:
import React, { useState, useEffect, useRef } from 'react';
import { Text, View, TouchableOpacity, StyleSheet } from 'react-native';
import { Camera } from 'expo-camera';
import * as MediaLibrary from 'expo-media-library';
import * as ImageManipulator from 'expo-image-manipulator';
import { MaterialIcons } from '@expo/vector-icons';
export default function CameraExample({ navigation }) {
const [hasPermission, setHasPermission] = useState(null);
const [type, setType] = useState(Camera.Constants.Type.back);
const cameraRef = useRef();
useEffect(() => {
(async () => {
const { status } = await Camera.requestCameraPermissionsAsync();
setHasPermission(status === 'granted');
})();
}, []);
const takePicture = async () => {
if (cameraRef.current) {
let photo = await cameraRef.current.takePictureAsync();
processPhoto(photo);
}
};
const processPhoto = async (photo) => {
try {
const { uri, width, height } = photo;
// Calculate the position to crop the image to maintain its center
const aspectRatio = width / height;
let cropWidth, cropHeight, cropX, cropY;
if (aspectRatio > 1) { // Landscape
cropWidth = height;
cropHeight = height;
cropX = (width - height) / 2;
cropY = 0;
} else if (aspectRatio < 1) { // Portrait
cropWidth = width;
cropHeight = width;
cropX = 0;
cropY = (height - width) / 2;
} else { // Square
cropWidth = width;
cropHeight = height;
cropX = 0;
cropY = 0;
}
// First, crop the image to a square while maintaining the center
const croppedImage = await ImageManipulator.manipulateAsync(
uri,
[
{
crop: {
originX: cropX,
originY: cropY,
width: cropWidth,
height: cropHeight,
}
},
],
{ format: ImageManipulator.SaveFormat.JPEG }
);
// Then resize the cropped image to be exactly 500x500
const resizedImage = await ImageManipulator.manipulateAsync(
croppedImage.uri,
[
{
resize: {
width: 500,
height: 500
}
},
],
{ format: ImageManipulator.SaveFormat.JPEG }
);
// Pass the processedUri to the CropScreen (Crop.js)
navigation.navigate('Crop', { imageUri: resizedImage.uri });
} catch (error) {
console.error('Error processing photo:', error);
}
};
if (hasPermission === null) {
return <View />;
}
if (hasPermission === false) {
return <Text>No access to camera</Text>;
}
return (
<View style={{ flex: 1 }}>
<Camera style={{ flex: 1 }} type={type} ref={cameraRef}>
<View style={styles.overlayContainer}>
<View style={styles.overlayTop} />
<View style={styles.overlayMiddle}>
<View style={styles.overlaySide} />
<View style={styles.overlayCenter} />
<View style={styles.overlaySide} />
</View>
<View style={styles.overlayBottom} />
</View>
<View style={styles.captureButtonContainer}>
<TouchableOpacity style={styles.captureButton} onPress={takePicture}>
<MaterialIcons name="camera" size={90} color="white" />
</TouchableOpacity>
</View>
</Camera>
</View>
);
}
const styles = StyleSheet.create({
captureButtonContainer: {
flex: 1,
flexDirection: 'row',
backgroundColor: 'transparent',
justifyContent: 'center',
alignItems: 'flex-end',
marginBottom: 20,
},
captureButton: {
borderRadius: 50,
backgroundColor: 'transparent',
},
overlayContainer: {
...StyleSheet.absoluteFillObject,
justifyContent: 'space-between',
},
overlayMiddle: {
flexDirection: 'row',
flex: 1,
},
overlayTop: {
flex: 1,
backgroundColor: 'rgba(128, 128, 128, 0.5)',
},
overlayBottom: {
flex: 1,
backgroundColor: 'rgba(128, 128, 128, 0.5)',
},
overlaySide: {
flex: 1,
backgroundColor: 'rgba(128, 128, 128, 0.5)',
},
overlayCenter: {
flex: 4,
borderColor: 'lightgrey',
borderWidth: 3,
},
});
code to process the image/ convert to tensor
import React, { useState, useEffect } from "react";
import { StyleSheet, View, TouchableOpacity, Text } from "react-native";
import * as tf from "@tensorflow/tfjs";
import { fetch, bundleResourceIO } from "@tensorflow/tfjs-react-native";
import Constants from "expo-constants";
import * as Permissions from "expo-permissions";
import * as ImagePicker from "expo-image-picker";
import * as jpeg from "jpeg-js";
import Output from "./Output";
import * as tf from '@tensorflow/tfjs'
import {bundleResourceIO, decodeJpeg} from '@tensorflow/tfjs-react-native'
import * as FileSystem from 'expo-file-system';
const modelJSON = require('assets/model.json')
const modelWeights = require('assets/weights.bin')
const loadModel = async()=>{
//.ts: const loadModel = async ():Promise<void|tf.LayersModel>=>{
const model = await tf.loadLayersModel(
bundleResourceIO(modelJSON, modelWeights)
).catch((e)=>{
console.log("[LOADING ERROR] info:",e)
})
return model
}
const transformImageToTensor = async (uri)=>{
//.ts: const transformImageToTensor = async (uri:string):Promise<tf.Tensor>=>{
//read the image as base64
const img64 = await FileSystem.readAsStringAsync(uri, {encoding:FileSystem.EncodingType.Base64})
const imgBuffer = tf.util.encodeString(img64, 'base64').buffer
const raw = new Uint8Array(imgBuffer)
let imgTensor = decodeJpeg(raw)
const scalar = tf.scalar(255)
//resize the image
imgTensor = tf.image.resizeNearestNeighbor(imgTensor, [300, 300])
//normalize; if a normalization layer is in the model, this step can be skipped
const tensorScaled = imgTensor.div(scalar)
//final shape of the rensor
const img = tf.reshape(tensorScaled, [1,300,300,3])
return img
}
const makePredictions = async ( batch, model, imagesTensor )=>{
//.ts: const makePredictions = async (batch:number, model:tf.LayersModel,imagesTensor:tf.Tensor<tf.Rank>):Promise<tf.Tensor<tf.Rank>[]>=>{
//cast output prediction to tensor
const predictionsdata= model.predict(imagesTensor)
//.ts: const predictionsdata:tf.Tensor = model.predict(imagesTensor) as tf.Tensor
let pred = predictionsdata.split(batch) //split by batch size
//return predictions
return pred
}
export const getPredictions = async (image)=>{
await tf.ready()
const model = await loadModel() as tf.LayersModel
const tensor_image = await transformImageToTensor(image)
const predictions = await makePredictions(1, model, tensor_image)
return predictions
}
code to print out the result
import React from 'react';
import { StyleSheet, Text, View, ScrollView } from 'react-native';
import CustomNavBar from './NavBar'; // Import the CustomNavBar component
import { getPredictions } from './modelload';
const PredResult = getPredictions(imageUri)
export default function Results({ navigation }) {
return (
<ScrollView>
<View style={styles.container}>
<Text style={styles.title}>Mole<Text style={styles.boldText}>Detect</Text></Text>
<View style={styles.circle}></View>
<View style={styles.topCircle}></View>
<View style={styles.bottomCircle}></View>
<View style={styles.boxesContainer}>
<View style={styles.leftAlignedBox}>
<View style={styles.headerBox}><Text style={styles.headerText}>Melanoma </Text></View>
<Text style={styles.contentText}>
If you have Melanoma: It often appears as an irregular, asymmetric mole with uneven borders, varying colors, and enlargement and / or bleeding.
</Text>
</View>
<View style={styles.rightAlignedBox}>
<View style={[styles.headerBox, styles.rightHeader]}><Text style={styles.headerText}>Further Steps</Text></View>
<Text style={styles.contentText}>
Visit <Text style={styles.linkText} onPress={() => Linking.openURL('https://www.cancer.gov/types/skin')}>NIH</Text> or <Text style={styles.linkText} onPress={() => Linking.openURL('https://www.cdc.gov/cancer/skin/basic_info/prevention.html')}>CDC</Text> for what to do next.
Please note that this is not an official diagnosis and you should consult a doctor for any serious doubts or possible skin conditions.
</Text>
</View>
</View>
<View style={styles.verticalLine}></View>
<CustomNavBar navigation={navigation} />
</View>
</ScrollView>
);
}
const styles = StyleSheet.create({
container: {
flex: 1,
backgroundColor: '#F8F8F8',
paddingHorizontal: 0,
paddingTop: 40,
},
boxesContainer: {
flex: 1,
},
boldText:{
fontWeight:'bold',
},
title: {
fontSize: 50,
marginBottom: 60,
alignSelf: 'center',
marginTop: 25,
},
circle: {
position: 'absolute',
right: 29, // Adjusted to horizontally center over the line
top: '35.5%',
width: 25, // Increased width
height: 25, // Increased height
borderRadius: 25, // Adjusted to keep the circle shape
backgroundColor: '#667799',
zIndex: 2,
},
topCircle: {
position: 'absolute',
right: 29,
top: '20%', // Position at the top of the line
width: 25,
height: 25,
borderRadius: 25,
backgroundColor: '#667799',
zIndex: 2,
},
bottomCircle: {
position: 'absolute',
right: 29,
top: '67.75%', // Position near the end of the line
width: 25,
height: 25,
borderRadius: 25,
backgroundColor: '#667799',
zIndex: 2,
},
verticalLine: {
position: 'absolute',
right: 40,
top: '20.5%', // Adjusted to connect to the bottom of the circle
height: '69.59%', // Adjusted to maintain its end position
width: 3,
backgroundColor: '#17364b',
zIndex: 1,
},
leftAlignedBox: {
width: '95%',
height: 250,
backgroundColor: '#e9dbd7',
marginBottom: 40,
paddingTop: 10, // Make space for the popped out header
},
rightAlignedBox: {
width: '95%',
height: 250,
backgroundColor: '#ced3df',
alignSelf: 'flex-end',
marginBottom:9,
paddingTop: 10, // Make space for the popped out header
},
headerBox: {
width: '50%',
backgroundColor: '#17364b',
padding: 10,
alignSelf: 'center',
marginTop: -25, // This will make the box pop out by half its height
marginRight: 60,
textAlign: 'center',
},
rightHeader: {
backgroundColor: '#17364b',
alignSelf: 'flex-end',
marginRight: 70,
},
headerText: {
fontSize: 22,
fontWeight: 'bold',
color: '#fff',
alignSelf: 'center',
},
contentText: {
margin: 10,
fontSize: 20,
color: '#333',
padding: 38,
marginTop: -10,
},
linkText: {
color: 'blue',
textDecorationLine: 'underline',
},
});
Instead of it printing melanoma i want it to print the result
I had tried to directly print the result but it was showing the command