Nodejs & Streaming API

[newbie99]

I'm attempting to login via the non-interactive login and access streaming via Node.js.

There was a useful example (which I now can't find) of someone logging into the Betting API via Node.js, which I managed to replicate, however the Streaming API is currently proving difficult.

I have also played around with Betfairlightweight (i.e. the Python wrapper) and from that and the Node.js example (that I can't find), I know my certs / credentials are correct.

However, if I attempt to access the streaming API, I get the following error:

Received: {"op":"connection","connectionId":"003-310119210425-261018"}

Received: {"op":"status","statusCode":"FAILURE","errorCode":"INVALID_SESSION_INFORMATION","errorMessage":"InvalidCredentials","connectionClosed":true,"connectionId":"003-310119210425-261018"}

If I'm interpreting that correctly then I believe my certificate / key are probably not being passed to the server correctly (however I may be wrong).

I'm hoping with all the smart people on here, someone will be able to spot my error and then I can continue to upload sample code as the project progresses.

Here is my code:

var https = require('https');
var fs = require('fs');
var tls = require('tls');

//'use strict';
const PORT = 443;
const HOST = 'stream-api.betfair.com';
// Pass the certs to the server and let it know to process even unauthorized certs.
var options = {
key: fs.readFileSync('FULL_FILE_PATH.key'),
cert: fs.readFileSync('FULL_FILE_PATH.crt')
};

var client = tls.connect(PORT, HOST, options, function() {

console.log("Connected");



client.on('data', function(data) {
console.log('Received: ' + data);


client.write('{"op": "authentication", "id": "1", "username": "USERNAME", "password": "PASSWORD", "appKey": "APPKEY", "session": "' + data.sessionToken + '"}\r\n');

});

});

I'm at a dead end here, so any pointers would be greatly appreciated.

[LiamP]

Why are you passing username / password? / is that session valid?

You login normally first, not sure why you are passing certs to the socket?

[newbie99]

Why are you passing username / password? / is that session valid?

You login normally first, not sure why you are passing certs to the socket?

You know what....I was thinking this as I went to bed!

I assumed you had to login to the stream....however it seems (and what you've said backs it up) that I need to login normally first, then obtain the session token and then try to connect to the stream.

I was treating it as a completely separate login, it was only through re-reading the docs and playing around with your Python code I started to realise, but thank you for clarifying, that should help....hopefully today I can make some progress!

[newbie99]

You know what....I was thinking this as I went to bed!

I assumed you had to login to the stream....however it seems (and what you've said backs it up) that I need to login normally first, then obtain the session token and then try to connect to the stream.

I was treating it as a completely separate login, it was only through re-reading the docs and playing around with your Python code I started to realise, but thank you for clarifying, that should help....hopefully today I can make some progress!

Okay, I have made some progress and now have streaming data appearing (for the avoidance of doubt I'm filtering on just one market during testing just to make it easier to manually read the data, not because I think its more efficient or anything like that, I appreciate its not).

However, I seem to have the same problem as before (when using Node.js), the data sometimes comes in multiple messages, which could make it tricky to send onto a web browser (which is the next phase).

Here is my code so far:

var https = require('https');
var fs = require('fs');
var url = require('url');
var tls = require('tls');

var uri = "https://identitysso-cert.betfair.com/api/certlogin";
var data = 'username=' + USERNAME + '&password=' + PASSWORD;

var options = url.parse(uri);
options.method = 'POST';
options.port = 443;
options.headers = {
    'Content-Type': 'application/x-www-form-urlencoded',
    'X-Application': 'test'
  };
options.key = fs.readFileSync(KEY);
options.cert = fs.readFileSync(CERT);
options.agent = new https.Agent(options);

var req = https.request(options, function(res) {
  console.log("statusCode:", res.statusCode);
  var responseData = "";
  res.on('data', function(d) {
     responseData += d;
  });
  res.on('end', function() {
     var response = JSON.parse(responseData);
     console.log("sessionToken:", response.sessionToken.replace(/\d/g, ''));

$st = response.sessionToken;

st($st);

  });
  res.on('error', function(e) {
    console.error(e);
  });
});

req.end(data);

function st($st) {

//'use strict'; 
const PORT = 443; 
const HOST = 'stream-api.betfair.com';

var client = tls.connect(PORT, HOST, function(st) { 

console.log("Connected");

client.write('{"op": "authentication", "appKey": "' + APPKEY + '", "session":"' + $st + '"}\r\n');
client.write('{"op":"marketSubscription","id":2,"marketFilter":{"marketIds":["1.130856098"]},"marketDataFilter":{}}\r\n');

client.on('data', function(data) {
console.log('Received: ' + data);


});


});

}

This 'works' in the sense, that the data appears, but as its split up, I suspect I can't pass it as a JSON onto a browser, hence I'm right back where I started with the example code from GitHub (albeit without a hard coded sesison token as at least now I can get that from the Cert login).

Can anyone spot anything obvious here, is it possibly my lack of understanding of the correct connection protocol for the streaming API?

[newbie99]

Okay, I have made some progress...I think I have now managed to come up with a work around for the fact the messages come through in parts:

var https = require('https');
var fs = require('fs');
var url = require('url');
var tls = require('tls');

var app = require('express')();
var http = require('http').Server(app);
var io = require('socket.io')(http);

var uri = "https://identitysso-cert.betfair.com/api/certlogin";
var data = 'username=' + USERNAME + '&password=' + PASSWORD;
$i=0;
$j=0;
var buffer='';
var a='';
var d='';

var options = url.parse(uri);
options.method = 'POST';
options.port = 443;
options.headers = {
    'Content-Type': 'application/x-www-form-urlencoded',
    'X-Application': 'test'
  };
options.key = fs.readFileSync(KEY);
options.cert = fs.readFileSync(CERT);
options.agent = new https.Agent(options);

var req = https.request(options, function(res) {
  console.log("statusCode:", res.statusCode);
  var responseData = "";
  res.on('data', function(d) {
     responseData += d;
  });
  res.on('end', function() {
     var response = JSON.parse(responseData);
     console.log("sessionToken:", response.sessionToken.replace(/\d/g, ''));

$st = response.sessionToken;

st($st);

  });
  res.on('error', function(e) {
    console.error(e);
  });
});

req.end(data);

function st($st) {

//'use strict'; 
const PORT = 443; 
const HOST = 'stream-api.betfair.com';

var client = tls.connect(PORT, HOST, function(st) { 

console.log("Connected");

client.write('{"op": "authentication", "appKey": "' + APPKEY + '", "session":"' + $st + '"}\r\n');
client.write('{"op":"marketSubscription","id":2,"marketFilter":{"marketIds":["1.130856098"]},"marketDataFilter":{}}\r\n');

client.on('data', function(data) {

//console.log('Received: ' + data);

const { StringDecoder } = require('string_decoder');
const decoder = new StringDecoder('utf8');
const utf_data = Buffer.from(data);
a = decoder.write(utf_data); 
var b = a.slice(-1);
//console.log('Received: ' + a);
//var c = b.replace(/(\r\n|\n|\r)/gm,"").trim();
var c = b.trim();
//console.log('Received: ' + '"' + c + '"');
$break = "\r\n";

if (c=="") {
    if ($j!=0) {$j=$i;} else {$j=$i;$i=0;}
    } else {$i++;}

if(($j + $i)>0) {$group = 'group';d+=a;} else {$group = 'ungroup';}

if ($group=='group') {if (c=="") {a=d;} else {a=null;}} else {a=a;}

buffer = a + $break;

console.log('Received: ' + c + '-' + $i + '-' + $j + '-' + $group);
console.log(buffer);

io.emit('test', JSON.parse(buffer));

});




});



client.on('close', function() {
    console.log('Connection closed');
});

client.on('error', function(err) {
    console.log('Error:' + err);
});

}


app.get('/', function(req, res) {
   res.sendfile('index.html');
});


http.listen(3000, function() {
   console.log('listening on localhost:3000');
});


function isString (value) {
return typeof value === 'string' || value instanceof String;
}

This seems to organise the messages into valid JSON's and then send onto the browser (index.html), however the problems I'm still finding are:

1) The first pricing message is always ignored (i.e. I can see it in the console, but it is never sent to the browser, I haven't worked out why yet)
2) This method is almost certainly inefficient and was more about me finding a quick fix, so there are probably issues I've missed
3) I don't believe its missing any messages, but its not impossible
4) I'm connecting to the betfair server via SSL and then to the browser via socket.io, ignoring the fact this isn't best practice (I presume), for personal use only (i.e. this isn't a commercial app), does it present any actual problems?

Any and all critique here is welcomed!

[newbie99]

All the above still apply and I thought I was going down the right route, using 'await'...however despite working most of the time, randomly it does appear to skip all messages, bar the Heartbeat ones (it could miss some of those too in fairness, I don't currently check to see if every single Heartbeat is sent, but in the browser console, I can see a load are).

Anyway, this is tonight's progress:

var https = require('https');
var fs = require('fs');
var url = require('url');
var tls = require('tls');

var app = require('express')();
var http = require('http').Server(app);
var io = require('socket.io')(http);

var uri = "https://identitysso-cert.betfair.com/api/certlogin";
var data = 'username=' + USERNAME + '&password=' + PASSWORD;
$i=0;
$j=0;
var buffer='';
var a='';
var b='';
var c='';
var d='';
$group ='';

var options = url.parse(uri);
options.method = 'POST';
options.port = 443;
options.headers = {
    'Content-Type': 'application/x-www-form-urlencoded',
    'X-Application': 'test'
  };
options.key = fs.readFileSync(KEY);
options.cert = fs.readFileSync(CERT);
options.agent = new https.Agent(options);

var req = https.request(options, function(res) {
  console.log("statusCode:", res.statusCode);
  var responseData = "";
  res.on('data', function(d) {
     responseData += d;
  });
  res.on('end', function() {
     var response = JSON.parse(responseData);
     console.log("sessionToken:", response.sessionToken.replace(/\d/g, ''));

$st = response.sessionToken;

st($st);

  });
  res.on('error', function(e) {
    console.error(e);
  });
});

req.end(data);

function st($st) {

//'use strict'; 
const PORT = 443; 
const HOST = 'stream-api.betfair.com';

var client = tls.connect(PORT, HOST, function(st) { 

console.log("Connected");

client.write('{"op": "authentication", "appKey": "' + APPKEY + '", "session":"' + $st + '"}\r\n');
client.write('{"op":"marketSubscription","id":2,"marketFilter":{"marketIds":["1.130856098"]},"marketDataFilter":{}}\r\n');

client.on('data', function(data) {

//console.log('Received: ' + data);

const { StringDecoder } = require('string_decoder');
const decoder = new StringDecoder('utf8');
const utf_data = Buffer.from(data);
a = decoder.write(utf_data); 
b = a.slice(-1);
//console.log('Received: ' + a);
//var c = b.replace(/(\r\n|\n|\r)/gm,"").trim();
c = b.trim();
//console.log('Received: ' + '"' + c + '"');
$break = "\r\n";


function_send_to_browser();


});


});


client.on('close', function() {
    console.log('Connection closed');
});

client.on('error', function(err) {
    console.log('Error:' + err);
});

}


app.get('/', function(req, res) {
   res.sendfile('index.html');
});


http.listen(3000, function() {
   console.log('listening on localhost:3000');
});

function function_a(){

if (c=="") {
    if ($j!=0) {$j=$i;} else {$j=$i;$i=0;}
    } else {$i++;};

return 'function complete';

}

async function function_b(){



if(($j + $i)>0) {$group = 'group';d+=a;} else {$group = 'ungroup';}

return 'function complete';

}

async function function_c() {



if (c=="") {if ($group=='group') {a=d;} else {a=a;} } else {a=null;}

return 'function complete';

}

async function function_buffer() {

if (a!==null) {
buffer = a + $break;

console.log('Received: ' + c + '-' + $i + '-' + $j + '-' + $group);
console.log(buffer);}

return 'buffer';

}

async function function_send_to_browser($buffer) {

const fnca = await function_a();
const fncb = await function_b();
const fncc = await function_c();
const fnc_buffer = await function_buffer();

io.emit('test', JSON.parse(buffer));

}

function isString (value) {
return typeof value === 'string' || value instanceof String;