How to Implement a tenserflow models prediction in a react native app page

35 views Asked by At

I was working on my app, and was trying to figure out how to get my model to output a result in my react native app.

In the app there is a page which prompts the user to take a photo, and from that image the model is supposed to print out the result on the next page after the user presses the button. I was having difficulty getting the model to output the result as I don't know where the image that the user had taken goes and how to use it. I was also have trouble finding out how to print the result in a certain font and text size as whenever I had tried to print the result it would print the command.

code to take the image:

import React, { useState, useEffect, useRef } from 'react';
import { Text, View, TouchableOpacity, StyleSheet } from 'react-native';
import { Camera } from 'expo-camera';
import * as MediaLibrary from 'expo-media-library';
import * as ImageManipulator from 'expo-image-manipulator';
import { MaterialIcons } from '@expo/vector-icons';

export default function CameraExample({ navigation }) {
  const [hasPermission, setHasPermission] = useState(null);
  const [type, setType] = useState(Camera.Constants.Type.back);
  const cameraRef = useRef();

  useEffect(() => {
    (async () => {
      const { status } = await Camera.requestCameraPermissionsAsync();
      setHasPermission(status === 'granted');
    })();
  }, []);

  const takePicture = async () => {
    if (cameraRef.current) {
      let photo = await cameraRef.current.takePictureAsync();
      processPhoto(photo);
    }
  };
  
  const processPhoto = async (photo) => {
    try {
      const { uri, width, height } = photo;
  
      // Calculate the position to crop the image to maintain its center
      const aspectRatio = width / height;
      let cropWidth, cropHeight, cropX, cropY;
      if (aspectRatio > 1) { // Landscape
        cropWidth = height;
        cropHeight = height;
        cropX = (width - height) / 2;
        cropY = 0;
      } else if (aspectRatio < 1) { // Portrait
        cropWidth = width;
        cropHeight = width;
        cropX = 0;
        cropY = (height - width) / 2;
      } else { // Square
        cropWidth = width;
        cropHeight = height;
        cropX = 0;
        cropY = 0;
      }
  
      // First, crop the image to a square while maintaining the center
      const croppedImage = await ImageManipulator.manipulateAsync(
        uri,
        [
          {
            crop: {
              originX: cropX,
              originY: cropY,
              width: cropWidth,
              height: cropHeight,
            }
          },
        ],
        { format: ImageManipulator.SaveFormat.JPEG }
      );
      
      // Then resize the cropped image to be exactly 500x500
      const resizedImage = await ImageManipulator.manipulateAsync(
        croppedImage.uri,
        [
          {
            resize: {
              width: 500,
              height: 500
            }
          },
        ],
        { format: ImageManipulator.SaveFormat.JPEG }
      );
  
      // Pass the processedUri to the CropScreen (Crop.js)
      navigation.navigate('Crop', { imageUri: resizedImage.uri });
    } catch (error) {
      console.error('Error processing photo:', error);
    }
  };
  
  

  if (hasPermission === null) {
    return <View />;
  }
  if (hasPermission === false) {
    return <Text>No access to camera</Text>;
  }

  return (
    <View style={{ flex: 1 }}>
      <Camera style={{ flex: 1 }} type={type} ref={cameraRef}>
        <View style={styles.overlayContainer}>
          <View style={styles.overlayTop} />
          <View style={styles.overlayMiddle}>
            <View style={styles.overlaySide} />
            <View style={styles.overlayCenter} />
            <View style={styles.overlaySide} />
          </View>
          <View style={styles.overlayBottom} />
        </View>
        <View style={styles.captureButtonContainer}>
          <TouchableOpacity style={styles.captureButton} onPress={takePicture}>
            <MaterialIcons name="camera" size={90} color="white" />
          </TouchableOpacity>
        </View>
      </Camera>
    </View>
  );
  }  

const styles = StyleSheet.create({
  captureButtonContainer: {
    flex: 1,
    flexDirection: 'row',
    backgroundColor: 'transparent',
    justifyContent: 'center',
    alignItems: 'flex-end',
    marginBottom: 20,
  },
  captureButton: {
    borderRadius: 50,
    backgroundColor: 'transparent',
  },
  overlayContainer: {
    ...StyleSheet.absoluteFillObject,
    justifyContent: 'space-between',
  },
  overlayMiddle: {
    flexDirection: 'row',
    flex: 1,
  },
  overlayTop: {
    flex: 1,
    backgroundColor: 'rgba(128, 128, 128, 0.5)',
  },
  overlayBottom: {
    flex: 1,
    backgroundColor: 'rgba(128, 128, 128, 0.5)',
  },
  overlaySide: {
    flex: 1,
    backgroundColor: 'rgba(128, 128, 128, 0.5)',
  },
  overlayCenter: {
    flex: 4,
    borderColor: 'lightgrey',
    borderWidth: 3,
  },
});

code to process the image/ convert to tensor


import React, { useState, useEffect } from "react";
import { StyleSheet, View, TouchableOpacity, Text } from "react-native";
import * as tf from "@tensorflow/tfjs";
import { fetch, bundleResourceIO } from "@tensorflow/tfjs-react-native";
import Constants from "expo-constants";
import * as Permissions from "expo-permissions";
import * as ImagePicker from "expo-image-picker";
import * as jpeg from "jpeg-js";
import Output from "./Output";
import * as tf from '@tensorflow/tfjs'
import {bundleResourceIO, decodeJpeg} from '@tensorflow/tfjs-react-native'
import * as FileSystem from 'expo-file-system';

const modelJSON = require('assets/model.json')
const modelWeights = require('assets/weights.bin')
const loadModel = async()=>{
  //.ts: const loadModel = async ():Promise<void|tf.LayersModel>=>{
      const model = await tf.loadLayersModel(
          bundleResourceIO(modelJSON, modelWeights)
      ).catch((e)=>{
        console.log("[LOADING ERROR] info:",e)
      })
      return model
  }
  const transformImageToTensor = async (uri)=>{
    //.ts: const transformImageToTensor = async (uri:string):Promise<tf.Tensor>=>{
    //read the image as base64
      const img64 = await FileSystem.readAsStringAsync(uri, {encoding:FileSystem.EncodingType.Base64})
      const imgBuffer =  tf.util.encodeString(img64, 'base64').buffer
      const raw = new Uint8Array(imgBuffer)
      let imgTensor = decodeJpeg(raw)
      const scalar = tf.scalar(255)
    //resize the image
      imgTensor = tf.image.resizeNearestNeighbor(imgTensor, [300, 300])
    //normalize; if a normalization layer is in the model, this step can be skipped
      const tensorScaled = imgTensor.div(scalar)
    //final shape of the rensor
      const img = tf.reshape(tensorScaled, [1,300,300,3])
      return img
  }
  const makePredictions = async ( batch, model, imagesTensor )=>{
    //.ts: const makePredictions = async (batch:number, model:tf.LayersModel,imagesTensor:tf.Tensor<tf.Rank>):Promise<tf.Tensor<tf.Rank>[]>=>{
    //cast output prediction to tensor
    const predictionsdata= model.predict(imagesTensor)
    //.ts: const predictionsdata:tf.Tensor = model.predict(imagesTensor) as tf.Tensor
    let pred = predictionsdata.split(batch) //split by batch size
    //return predictions 
    return pred
}

export const getPredictions = async (image)=>{
  await tf.ready()
  const model = await loadModel() as tf.LayersModel
  const tensor_image = await transformImageToTensor(image)
  const predictions = await makePredictions(1, model, tensor_image)
  return predictions    
}

code to print out the result

import React from 'react';
import { StyleSheet, Text, View, ScrollView } from 'react-native';
import CustomNavBar from './NavBar'; // Import the CustomNavBar component
import { getPredictions } from './modelload';
const PredResult = getPredictions(imageUri)
export default function Results({ navigation }) {
  return (
    <ScrollView>
    <View style={styles.container}>
      <Text style={styles.title}>Mole<Text style={styles.boldText}>Detect</Text></Text>

      <View style={styles.circle}></View> 
      <View style={styles.topCircle}></View>
      <View style={styles.bottomCircle}></View> 

      <View style={styles.boxesContainer}>
        <View style={styles.leftAlignedBox}> 
          <View style={styles.headerBox}><Text style={styles.headerText}>Melanoma </Text></View>
          <Text style={styles.contentText}>
          If you have Melanoma: It often appears as an irregular, asymmetric mole with uneven borders, varying colors, and enlargement and / or bleeding.
          </Text>
        </View>

        <View style={styles.rightAlignedBox}>
          <View style={[styles.headerBox, styles.rightHeader]}><Text style={styles.headerText}>Further Steps</Text></View>
          <Text style={styles.contentText}>
            Visit <Text style={styles.linkText} onPress={() => Linking.openURL('https://www.cancer.gov/types/skin')}>NIH</Text> or <Text style={styles.linkText} onPress={() => Linking.openURL('https://www.cdc.gov/cancer/skin/basic_info/prevention.html')}>CDC</Text> for what to do next.
            Please note that this is not an official diagnosis and you should consult a doctor for any serious doubts or possible skin conditions.
          </Text>
        </View>
      </View>

      <View style={styles.verticalLine}></View> 
      <CustomNavBar navigation={navigation} />
    </View>
    </ScrollView>
  );
}
  
  const styles = StyleSheet.create({
    container: {
      flex: 1,
      backgroundColor: '#F8F8F8',
      paddingHorizontal: 0,
      paddingTop: 40,
    },
    boxesContainer: {
      flex: 1,
    },
    boldText:{
      fontWeight:'bold',
    },
    title: {
      fontSize: 50,
      marginBottom: 60,
      alignSelf: 'center',
      marginTop: 25,
    },
    circle: {
      position: 'absolute',
      right: 29,  // Adjusted to horizontally center over the line
      top: '35.5%', 
      width: 25,  // Increased width
      height: 25, // Increased height
      borderRadius: 25, // Adjusted to keep the circle shape
      backgroundColor: '#667799',
      zIndex: 2,
    },
    topCircle: {
      position: 'absolute',
      right: 29,
      top: '20%',  // Position at the top of the line
      width: 25,
      height: 25,
      borderRadius: 25,
      backgroundColor: '#667799',
      zIndex: 2,
    },
    bottomCircle: {
      position: 'absolute',
      right: 29,
      top: '67.75%',  // Position near the end of the line
      width: 25,
      height: 25,
      borderRadius: 25,
      backgroundColor: '#667799',
      zIndex: 2,
    },
    verticalLine: {
      position: 'absolute',
      right: 40,
      top: '20.5%', // Adjusted to connect to the bottom of the circle
      height: '69.59%', // Adjusted to maintain its end position
      width: 3,
      backgroundColor: '#17364b',
      zIndex: 1,
    },
      leftAlignedBox: {
        width: '95%',
        height: 250,
        backgroundColor: '#e9dbd7',
        marginBottom: 40,
        paddingTop: 10, // Make space for the popped out header
      },
      rightAlignedBox: {
        width: '95%',
        height: 250,
        backgroundColor: '#ced3df',
        alignSelf: 'flex-end',
        marginBottom:9,
        paddingTop: 10, // Make space for the popped out header
        
      },
      headerBox: {
        width: '50%',
        backgroundColor: '#17364b',
        padding: 10,
        alignSelf: 'center',
        marginTop: -25, // This will make the box pop out by half its height
        marginRight: 60,
        textAlign: 'center',
      },
      rightHeader: {
        backgroundColor: '#17364b',
        alignSelf: 'flex-end',
        marginRight: 70,
      },
      headerText: {
        fontSize: 22,
        fontWeight: 'bold',
        color: '#fff',
        alignSelf: 'center',
      },
      contentText: {
        margin: 10,
        fontSize: 20,
        color: '#333',
        padding: 38,
        marginTop: -10,
      },
      linkText: {
        color: 'blue',
        textDecorationLine: 'underline',
      },
});

Instead of it printing melanoma i want it to print the result

I had tried to directly print the result but it was showing the command

0

There are 0 answers