サンプルコード
GAS
function doPost(e) {
var a = [e.postData.getDataAsString(),'en','ja']
return ContentService.createTextOutput(LanguageApp.translate(...a))
}
MainWindow.xaml.cs
namespace Konjac{
public partial class MainWindow : Window{
public MainWindow(){
InitializeComponent();
webView.NavigationCompleted += (s, e) => {
webView.CoreWebView2.AddHostObjectToScript("bridge", new Bridge(webView));
};
webView.Source = new Uri(GetPath() + "\\Konjac.html");
}
private static string GetPath(){
var p = System.Diagnostics.Process.GetCurrentProcess();
return System.IO.Path.GetDirectoryName(p.MainModule.FileName);
}
}
}
Bridge.cs
public class Native{
// DPI (PerMonitorV2)
//https://msdn.microsoft.com/en-us/library/windows/desktop/dd145062(v=vs.85).aspx
//https://msdn.microsoft.com/en-us/library/windows/desktop/dn280510(v=vs.85).aspx
public const int WM_DPICHANGED = 0x02E0;
[DllImport("User32.dll")]
public static extern IntPtr MonitorFromPoint([In] Point pt, [In] uint dwFlags);
[DllImport("Shcore.dll")]
public static extern IntPtr GetDpiForMonitor([In] IntPtr hmonitor, [In] DpiType dpiType, [Out] out uint dpiX, [Out] out uint dpiY);
public enum DpiType{
Effective = 0,
Angular = 1,
Raw = 2,
}
}
public class ScreenData{
public uint dpi { get; set; }
public Screen screen { get; set; }
public ScreenData(Screen s){
var p = new Point(s.Bounds.Left + 1, s.Bounds.Top + 1);
var m = Native.MonitorFromPoint(p, 2/*MONITOR_DEFAULTTONEAREST*/);
uint x, y;
Native.GetDpiForMonitor(m, Native.DpiType.Effective, out x, out y);
screen = s;
dpi = x;
}
}
public class BridgeEvent{
public string type { get; set; }
public string value { get; set; }
public BridgeEvent(string t, string v){
type = t;
value = v;
}
public string ToJson(){
return JsonSerializer.Serialize(this);
}
}
[ComVisible(true)]
public class Bridge{
Bitmap capture;
WebView2 view;
public Bridge(WebView2 w){
view = w;
}
public async void Scan(int x, int y, int w, int h){
Bitmap textBmp = new Bitmap(w, h);
Graphics g = Graphics.FromImage(textBmp);
g.DrawImage(capture, new Point(-x, -y));
g.Dispose();
string file = "Konjac.png";
string path = $"{GetPath()}\\{file}";
textBmp.Save(path);
string s = await OCR(file);
Console.WriteLine(s);
var e = new BridgeEvent("SCAN_DONE", s);
//jsからの非同期呼び出しでnullが返る
//https://github.com/MicrosoftEdge/WebView2Feedback/issues/822
view.CoreWebView2.PostWebMessageAsString(e.ToJson());
}
public async Task<string> OCR(string file){
//https://water2litter.net/rye/post/c_ocr_hello/
Stream input_stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read);
var source = new WriteableBitmap(System.Windows.Media.Imaging.BitmapFrame.Create(input_stream));
input_stream.Close();
PngBitmapEncoder encoder = new PngBitmapEncoder();
encoder.Frames.Add(System.Windows.Media.Imaging.BitmapFrame.Create(source));
MemoryStream temp_stream = new MemoryStream();
encoder.Save(temp_stream);
var converted_stream = WindowsRuntimeStreamExtensions.AsRandomAccessStream(temp_stream);
var decorder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(converted_stream);
SoftwareBitmap bitmap = await decorder.GetSoftwareBitmapAsync();
converted_stream.Dispose();
temp_stream.Close();
OcrEngine engine = OcrEngine.TryCreateFromLanguage(new Windows.Globalization.Language("en"));
var result = await engine.RecognizeAsync(bitmap);
return result.Text;
}
public string Post(string url, string data){
//https://docs.microsoft.com/ja-jp/dotnet/framework/network-programming/how-to-send-data-using-the-webrequest-class
Console.WriteLine(url);
byte[] byteArray = Encoding.UTF8.GetBytes(data);
var request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.ContentLength = byteArray.Length;
Stream dataStream = request.GetRequestStream();
dataStream.Write(byteArray, 0, byteArray.Length);
dataStream.Close();
var response = (HttpWebResponse)request.GetResponse();
string responseContent = string.Empty;
using (dataStream = response.GetResponseStream())
{
StreamReader reader = new StreamReader(dataStream);
string responseFromServer = reader.ReadToEnd();
Console.WriteLine(responseFromServer);
response.Close();
return responseFromServer;
}
}
public string CaptureScreen(int i){
Screen s = Screen.AllScreens[i];
capture = new Bitmap(s.Bounds.Width, s.Bounds.Height);
Graphics g = Graphics.FromImage(capture);
g.CopyFromScreen(new Point(s.Bounds.X, s.Bounds.Y), new Point(0, 0), capture.Size);
g.Dispose();
string file = $"Screen{i}.png";
string path = $"{GetPath()}\\{file}";
Console.WriteLine(path);
capture.Save(path);
return file;
}
private static string GetPath(){
var p = System.Diagnostics.Process.GetCurrentProcess();
return Path.GetDirectoryName(p.MainModule.FileName);
}
public string getAllScreens(){
List<ScreenData> screens = new List<ScreenData>();
foreach (Screen s in Screen.AllScreens){
screens.Add(new ScreenData(s));
}
return JsonSerializer.Serialize(screens);
}
}
Konjac.html
<!DOCTYPE html><html lang="ja"><head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
<script>
DEPLOY_ID = '***************************' //GAS
$(e=>{$('body').append(new Konjac().view)})
class Konjac{
constructor(){
Konjac.native = new Konjac.Bridge().ready(v=>{
v.getScreens(r=>{
this.item = r.map((v,i)=>new Konjac.Screen(v,i))
this.init()
})
})
this.view = _html('.container')
}
init(){
this.currentItem = this.item[0]
var screens = this.item.sort((a,b)=>{return a.x-b.x})
screens.forEach(v=>v.menuItem.click(e=>this.select(v)))
this.view.append(
_html('.content',screens.map(v=>v.view)),
this.overlay = _html('.overlay').hide().click(e=>this.overlay.hide()),
_html('.header',
_html('ul',screens.map(v=>v.menuItem)),
_html('.button,翻訳する').click(e=>{
this.overlay.empty().show()
this.ocr()
})
),
)
Konjac.native.hook.SCAN_DONE = e=>{
this.overlay.append(_html('.en',e.value))
this.translate(e.value,ja=>{
this.overlay.prepend(_html('.ja',ja))
})
}
var g = new Konjac.Gamepad({
GAMEPAD_A:e=>this.rescan(),
GAMEPAD_B:e=>{this.overlay.hide()}
})
this.select(this.currentItem)
}
rescan(){
this.currentItem.capture(r=>{
this.overlay.empty().show()
this.ocr()
})
}
select(item){
this.overlay.hide()
this.currentItem = item
this.item.forEach(v=>{
v.menuItem.removeClass('selected')
v.view.hide()
})
item.menuItem.addClass('selected')
item.view.show()
item.capture()
}
ocr(){
var dpr = this.currentItem.dpr
var w = $(window)
var rect = [
Math.round(w.scrollLeft()*dpr),
Math.round(w.scrollTop()*dpr),
Math.round(w.width()*dpr),
Math.round((w.height()-$('.header').height())*dpr),
]
Konjac.native.ocr(...rect)
return this
}
translate(data,success){
var url = `https://script.google.com/macros/s/${DEPLOY_ID}/exec`;
Konjac.native.post(url,data,v=>success(v))
return this
}
}
Konjac.Gamepad = class KonjacGamepad{
constructor(hook){
var s = 'A,B,X,Y,LB,RB,LT,RT,BACK,START,LS,RS,UP,DOWN,LEFT,RIGHT,HOME'
this.types = s.split(',').map(s=>'GAMEPAD_'+s)
this.hook = hook
this.stack = []
this.length = 0
window.addEventListener("gamepadconnected",e=>{
console.log(e)
this.stack[e.gamepad.index] = [e.gamepad]
this.length++
});
window.addEventListener("gamepaddisconnected",e=>{
console.log(e)
this.stack[e.gamepad.index] = []
this.length--
});
setInterval(e=>this.loop(),1000/30)
}
loop(){
if(this.length==0) return
var items = [...navigator.getGamepads()].filter(Boolean)
if(items.length==0) return
var a = this.stack[items[0].index]
a.unshift(items[0])
a[0].buttons.forEach((b,i)=>{
if(b.pressed==true) return
if(a[1].buttons[i].pressed==false) return
this.dispatch(this.types[i])
})
this.stack[items[0].index] = a.slice(0,1)
}
dispatch(type){
console.log(type)
if(this.hook[type]) this.hook[type]()
}
}
Konjac.Screen = class KonjacScreen{
constructor(data,i){
var b = data.screen.Bounds
var dpr = data.dpi/96
this.x = b.Left
this.y = b.Top
this.w = b.Width
this.h = b.Height
this.width = Math.round(b.Width/dpr)
this.height = Math.round(b.Height/dpr)
this.dpr = dpr//devicePixelRatio
this.name = data.screen.DeviceName.replace(/[^\w\s]/g,'')
this.index = i
Object.assign(this,data)
var s = b.Width+' x '+b.Height
this.sizeText = Konjac.Screen.type[s]||s
this.menuItem = _html('li',this.name,' ',_html('span',this.sizeText))
this.view = _html('img').css({width:this.width})
}
capture(f){
Konjac.native.capture(this.index,r=>{
this.view.attr('src',r+'?'+Math.random())
if(f) f(r)
})
return this
}
}
Konjac.Screen.type = {
'1920 x 1080':'FHD',
'1920 x 1200':'WUXGA',
'2560 x 1440':'WQHD',
'3840 x 2160':'4K',
}
Konjac.Bridge = class KonjacBridge{
constructor(){
this.hook = {}
this.on(()=>{}).ready(()=>{})
this.hostObjects = {bridge:{}}
if(chrome.webview){
this.hostObjects = chrome.webview.hostObjects
chrome.webview.addEventListener('message',e=>{
var data = JSON.parse(e.data)
var hook = this.hook[data.type]||this.hook.on
hook(data)
})
}
setTimeout(e=>this.hook.ready(this),100)
}
on(f){this.hook.on = f
return this
}
ready(f){this.hook.ready = f
return this
}
getScreens(f){
this.hostObjects.bridge.getAllScreens().then(r=>{
f(JSON.parse(r))
}).catch(console.log)
return this
}
capture(i,f){
this.hostObjects.bridge.CaptureScreen(i).then(r=>f(r))
return this
}
ocr(...a){this.hostObjects.bridge.Scan(...a)
return this
}
async post(url,data,success) {
var result = await this.hostObjects.bridge.Post(url,data);
success(result)
}
}
function _html(...a){
var b = a.shift().split(',')
var c = b.shift().split('.')
var d = c.shift()||'div'
return $(`<${d}>`).addClass(...c).append(...b,...a)
}
</script>
<style>
/* 略 */
</style></head></html>
↧