using UnityEngine;
using System.Collections;
using UnityEngine.Networking;
using System;
using TMPro;
using Newtonsoft.Json;
using System.Collections.Generic;
public class ImageRequester : MonoBehaviour
{
private string apiURL = "https://api.openai.com/v1/chat/completions";
private string apiKey = "APIKey";
public Camera mainCamera;
public RenderTexture renderTexture;
public TMP_Text tmpText;
public bool TextVisible = false;
private List<Message> conversationHistory = new List<Message>();
[System.Serializable]
public class ChatRequest
{
public string model = "gpt-4-vision-preview";
public Message[] messages;
public int max_tokens = 300;
}
[System.Serializable]
public class Message
{
public string role;
public Content[] content;
}
[Serializable]
public class Content
{
public string type;
public string text;
public ImageURL image_url;
}
[Serializable]
public class ImageURL
{
public string url;
}
[Serializable]
public class ResponseData
{
public Choice[] choices;
[Serializable]
public class Choice
{
public Message message;
}
[Serializable]
public class Message
{
public string content;
}
}
private string ParseResponse(string response)
{
try
{
ResponseData data = JsonConvert.DeserializeObject<ResponseData>(response);
if (data != null && data.choices != null && data.choices.Length > 0 && data.choices[0].message != null)
{
return data.choices[0].message.content;
}
}
catch (Exception e)
{
Debug.LogError("Error parsing response: " + e.Message);
}
return string.Empty;
}
void Start()
{
conversationHistory.Add(new Message
{
role = "system",
content = new[]
{
new Content { type = "text", text = "あなたは親切なアシスタントです。" }
}
});
}
public void RequestButtonClicked(string prompt)
{
if (tmpText.IsActive())
{
tmpText.gameObject.SetActive(false);
}
else
{
tmpText.gameObject.SetActive(true);
tmpText.text = "";
ImageRequestToGPT(prompt);
}
}
public void ImageRequestToGPT(string prompt)
{
Debug.Log("Button Clicked. Requesting image description...");
mainCamera.targetTexture = renderTexture;
mainCamera.Render();
RenderTexture.active = renderTexture;
Texture2D image = new Texture2D(renderTexture.width, renderTexture.height);
image.ReadPixels(new Rect(0, 0, renderTexture.width, renderTexture.height), 0, 0);
image.Apply();
tmpText.text = "Start requesting image description...";
byte[] imageBytes = image.EncodeToPNG();
string base64Image = Convert.ToBase64String(imageBytes);
mainCamera.targetTexture = null;
RenderTexture.active = null;
StartCoroutine(CallOpenAIVisionAPI(prompt,base64Image));
Destroy(image);
}
IEnumerator CallOpenAIVisionAPI(string prompt, string base64Image)
{
var request = new UnityWebRequest(apiURL, "POST");
Message newMessage = new Message
{
role = "user",
content = new[]
{
new Content { type = "text", text = prompt },
new Content { type = "image_url", image_url = new ImageURL{url=$"data:image/png;base64,{base64Image}" } }
}
};
conversationHistory.Add(newMessage);
Message[] messagesToSend = conversationHistory.ToArray();
ChatRequest chatRequest = new ChatRequest { messages = messagesToSend };
var settings = new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore
};
string jsonPayload = JsonConvert.SerializeObject(chatRequest, settings);
Debug.Log("jsonPayload" + jsonPayload);
request.uploadHandler = new UploadHandlerRaw(System.Text.Encoding.UTF8.GetBytes(jsonPayload));
request.downloadHandler = new DownloadHandlerBuffer();
request.SetRequestHeader("Content-Type", "application/json");
request.SetRequestHeader("Authorization", $"Bearer {apiKey}");
yield return request.SendWebRequest();
if (request.isNetworkError || request.isHttpError)
{
Debug.LogError($"Error: {request.error}");
}
else
{
string description = ParseResponse(request.downloadHandler.text);
Debug.Log("Description: " + description);
tmpText.text = description;
}
}
}
