//+------------------------------------------------------------------+
//|                                                      WinInet.mqh |
//|                                               Paul Hampton-Smith |
//|                                               paul1000@pobox.com |
//+------------------------------------------------------------------+

// This include has been adapted from the groundbreaking GrabWeb script 
// developed by Adbi that showed me how to import functions from wininet.dll
//+------------------------------------------------------------------+
//|                                                      grabweb.mq4 |
//|                                           Copyright © 2006, Abhi |
//|                                http://www.megadelfi.com/experts/ |
//|                            E-mail: grabwebexpert{Q)megadelfi.com |
//|                       fix my e-mail address before mailing me ;) |
//+------------------------------------------------------------------+


// Main Webscraping function
// ~~~~~~~~~~~~~~~~~~~~~~~~~
// bool GrabWeb(string strUrl, string& strWebPage)
// returns the text of any webpage. Returns false on timeout or other error
// 
// Parsing functions
// ~~~~~~~~~~~~~~~~~
// string GetData(string strWebPage, int nStart, string strLeftTag, string strRightTag, int& nPos)
// obtains the text between two tags found after nStart, and sets nPos to the end of the second tag
//
// void Goto(string strWebPage, int nStart, string strTag, int& nPos)
// Sets nPos to the end of the first tag found after nStart 


#include <LogUtils.mqh>

bool bWinInetDebug = false;

int hSession_IEType;
int hSession_Direct;
int Internet_Open_Type_Preconfig = 0;
int Internet_Open_Type_Direct = 1;
int Internet_Open_Type_Proxy = 3;
int Buffer_LEN = 13;

#import "wininet.dll"

#define INTERNET_FLAG_PRAGMA_NOCACHE    0x00000100 // Forces the request to be resolved by the origin server, even if a cached copy exists on the proxy.
#define INTERNET_FLAG_NO_CACHE_WRITE    0x04000000 // Does not add the returned entity to the cache. 
#define INTERNET_FLAG_RELOAD            0x80000000 // Forces a download of the requested file, object, or directory listing from the origin server, not from the cache.

int InternetOpenA(
	string sAgent,
   int lAccessType,
   string sProxyName="",
   string sProxyBypass="",
   int lFlags=0
);

int InternetOpenUrlA(
	int hInternetSession,
   string sUrl, 
   string sHeaders="",
   int lHeadersLength=0,
   int lFlags=0,
   int lContext=0 
);

int InternetReadFile(
	int hFile,
   string sBuffer,
   int lNumBytesToRead,
   int& lNumberOfBytesRead[]
);

int InternetCloseHandle(
	int hInet
);
#import

int hSession(bool Direct)
{
	string InternetAgent;
   if (hSession_IEType == 0)
   {
   	InternetAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461)";
      hSession_IEType = InternetOpenA(InternetAgent, Internet_Open_Type_Preconfig, "0", "0", 0);
      hSession_Direct = InternetOpenA(InternetAgent, Internet_Open_Type_Direct, "0", "0", 0);
   }
   if (Direct) 
   { 
   	return(hSession_Direct); 
   }
   else 
   {
   	return(hSession_IEType); 
  	}
}

bool GrabWeb(string strUrl, string& strWebPage)
{
	int hInternet;
	int iResult;
	int lReturn[]={1};
	string sBuffer="x";
	int bytes;
	hInternet = InternetOpenUrlA(hSession(FALSE), strUrl, "0", 0, INTERNET_FLAG_NO_CACHE_WRITE | INTERNET_FLAG_PRAGMA_NOCACHE | INTERNET_FLAG_RELOAD, 0);
	if (bWinInetDebug) Log("hInternet: " + hInternet);   
	if (hInternet == 0) return(false);
	iResult = InternetReadFile(hInternet, sBuffer, Buffer_LEN, lReturn);
	if (bWinInetDebug) Log("iResult: " + iResult);
	if (bWinInetDebug) Log("lReturn: " + lReturn[0]);
	if (bWinInetDebug) Log("iResult: " + iResult);
	if (bWinInetDebug) Log("sBuffer: " +  sBuffer);
	if (iResult == 0) return(false);
	bytes = lReturn[0];

   strWebPage = StringSubstr(sBuffer, 0, lReturn[0]);
   //if there's more data then keep reading it into the buffer
   while (lReturn[0] != 0)
   {
   	iResult = InternetReadFile(hInternet, sBuffer, Buffer_LEN, lReturn);
      if (lReturn[0]==0) break;
      bytes = bytes + lReturn[0];
      strWebPage = strWebPage + StringSubstr(sBuffer, 0, lReturn[0]);
   }

	iResult = InternetCloseHandle(hInternet);
	if (iResult == 0) return(false);
	if (bWinInetDebug) 
	{ 	
		Log("iResult: " + iResult);
		int handle = FileOpen("Grabweb.htm",FILE_BIN|FILE_READ|FILE_WRITE);
	   FileWriteString(handle,strWebPage,StringLen(strWebPage));
   	FileClose(handle);
   }
   return(true);
}

string GetData(string strWebPage, int nStart, string strLeftTag, string strRightTag, int& nPos)
{
	nPos = StringFind(strWebPage,strLeftTag,nStart);
	if (nPos == -1)
	{
		if (bWinInetDebug) Log("GetData(WebPage,"+nStart+","+strLeftTag+","+strRightTag+"): not found, returning empty string and -1");
		return("");
	}
	nPos += StringLen(strLeftTag);
	int nEnd = StringFind(strWebPage,strRightTag,nPos+1);
	string strData = StringTrimLeft(StringTrimRight(StringSubstr(strWebPage,nPos,nEnd-nPos)));
	if (bWinInetDebug) Log("GetData(WebPage,"+nStart+","+strLeftTag+","+strRightTag+") returning "+strData+" and "+nPos);
	return(strData);
}

void Goto(string strWebPage, int nStart, string strTag, int& nPos)
{
	nPos = StringFind(strWebPage,strTag,nStart);
	if (nPos == -1) 
	{
		if (bWinInetDebug) Log("Goto("+nStart+","+strTag+") not found. Returning -1");
		return;
	}
	nPos += StringLen(strTag);
	if (bWinInetDebug) Log("Goto("+nStart+","+strTag+") returning "+nPos);
}

void SaveWebPage(string strWebPage, string strID)
{
	string strFilename = strID+"_"+Year()+Pad(Month())+Pad(Day())+"-"+Pad(Hour())+Pad(Minute())+Pad(Seconds())+".htm";
	int handle = FileOpen(strFilename,FILE_BIN|FILE_WRITE);
	FileWriteString(handle,strWebPage,StringLen(strWebPage));
   FileClose(handle);
}

string Pad(int n)
{
	string strPad = "";
	if (n <= 9) strPad = "0";
	return(strPad+n);
}