Skip to content

eclipse/issue/32 #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

eclipse/issue/32 #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fdce03a
feat(webui): allow user to configure contentEncoding
evlach May 7, 2020
3c9a479
feat(Consumer): decode incoming message based on stream config conten…
evlach May 7, 2020
435abc3
feat(MqttConsumer): instead of passing js object to sdk, pass buffer …
evlach May 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/sdk-streams/src/ConsumerMixin.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ const ConsumerMixin = (Connector) =>
// );
let msg = {};
try {
msg = Utils.transformToJSONObject(m, this.config.mimeType);
msg = Utils.transformToJSONObject(m, this.config);
} catch (err) {
msg.error = err.message;
}
Expand Down
14 changes: 13 additions & 1 deletion packages/sdk-streams/src/configurations/ConsumerConfiguration.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ class ConsumerConfiguration extends BaseConfiguration {
if (config.mimeType) {
this.mimeType = config.mimeType || 'auto';
}
if (config.contentEncoding) {
this.contentEncoding = config.contentEncoding || 'utf8';
}
if (this.provider && this.provider.definition) {
config = config || {};
this.fields = this.fields || {};
Expand Down Expand Up @@ -72,7 +75,8 @@ class ConsumerConfiguration extends BaseConfiguration {
idAttribute: this.idAttribute || undefined,
samplePayloads: this.samplePayloads || undefined,
providerId: this.provider.id,
mimeType: this.mimeType || 'auto'
mimeType: this.mimeType || 'auto',
contentEncoding: this.contentEncoding || 'utf8',
});
if (this.provider && this.provider.definition) {
this.provider.definition.consumer.forEach((d) => {
Expand Down Expand Up @@ -136,6 +140,14 @@ class ConsumerConfiguration extends BaseConfiguration {
this._mimeType = value;
}

get contentEncoding() {
return this._contentEncoding;
}

set contentEncoding(value) {
this._contentEncoding = value;
}

get idAttribute() {
return this._idAttribute;
}
Expand Down
15 changes: 11 additions & 4 deletions packages/sdk-streams/src/helpers/Utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,18 @@ module.exports = class Utils {
* @returns {*}
*/

static transformToJSONObject(message, mimeType = 'application/json') {
static transformToJSONObject(message, config) {
const {mimeType = 'application/json', contentEncoding = 'utf8'} = config;
// TODO: revisit if/when force message to be always of Buffer type
if (!message) return message;
const msgString = Buffer.isBuffer(message)
? message.toString()
: message;
let msgString = '';
if(Buffer.isBuffer(message)) {
msgString = Buffer.isBuffer(message)
? message.toString(contentEncoding)
: message;
} else if(typeof message === 'string') {
msgString = Buffer.from(message).toString(contentEncoding);
}
if (typeof msgString === 'object') return msgString;
let msg = {};
switch (mimeType) {
Expand Down
39 changes: 33 additions & 6 deletions packages/sdk-streams/test/helpers/Utils.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,44 @@ describe('Stream API#Utils', () => {

it('should transform a Buffer/text to json object and back', () => {
const text = Buffer.from('Happy');
const jsonObject = Utils.transformToJSONObject(text, 'text/plain');
const jsonObject = Utils.transformToJSONObject(text, {
mimeType: 'text/plain'
});
expect(typeof jsonObject === 'object');
expect(jsonObject.text === 'Happy');
const text_ = Utils.transformFromJSONObject(jsonObject, 'text/plain');
const text_ = Utils.transformFromJSONObject(jsonObject, {
mimeType: 'text/plain'
});
expect(text_ === text);
expect(text_.length === text.length);
});

it('should transform a Buffer to json object based on contentType', () => {
const hexInput = '73747265616d736865657473';
const utf8Out = 'streamsheets';
const buff = Buffer.from(utf8Out);
const jsonObjectHex = Utils.transformToJSONObject(buff, {
mimeType: 'text/plain',
contentEncoding: 'hex'
});
expect(typeof jsonObjectHex === 'object');
expect(jsonObjectHex.value === hexInput);
const jsonObjectUtf8 = Utils.transformToJSONObject(buff, {
mimeType: 'text/plain',
});
expect(typeof jsonObjectHex === 'object');
expect(jsonObjectUtf8.value === hexInput);
});

it('should transform a json to json object and back', () => {
const json = JSON.stringify({
text: 'Happy',
num: 3
});
const jsonObject = Utils.transformToJSONObject(
json,
'application/json'
json,{
mimeType: 'application/json'
}
);
expect(typeof jsonObject === 'object');
expect(jsonObject.text === 'Happy');
Expand All @@ -45,7 +68,9 @@ describe('Stream API#Utils', () => {
' <todo>Work</todo>' +
' <todo>Play</todo>' +
'</note>';
const jsonObject = Utils.transformToJSONObject(xml, 'application/xml');
const jsonObject = Utils.transformToJSONObject(xml, {
mimeType: 'application/xml'
});
expect(typeof jsonObject === 'object');
expect(jsonObject.note.title._text === 'Happy');
expect(jsonObject.note.num._text === 3);
Expand All @@ -65,7 +90,9 @@ describe('Stream API#Utils', () => {
' <todo>Work</todo>' +
' <todo>Play</todo>' +
'</note>';
const jsonObject = Utils.transformToJSONObject(xml, 'auto');
const jsonObject = Utils.transformToJSONObject(xml, {
mimeType: 'auto'
});
expect(typeof jsonObject === 'object');
expect(jsonObject.note.title._text === 'Happy');
expect(jsonObject.note.num._text === 3);
Expand Down
5 changes: 3 additions & 2 deletions packages/stream/mqtt/src/MqttConsumer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ module.exports = class MqttConsumer extends sdk.ConsumerMixin(MqttConnector) {
registerDefaultListeners() {
super.registerDefaultListeners();
this._client.on('message', (topic, message, packet) => {
this.onMessage(topic, message, {
userProperties: packet.properties ? packet.properties.userProperties : {}
const { payload, properties } = packet;
this.onMessage(topic, payload, {
userProperties: properties ? properties.userProperties : {}
});
});
}
Expand Down
40 changes: 40 additions & 0 deletions packages/webui/src/components/Admin/streams/StreamForm.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,46 @@ class StreamForm extends Component {
}),
model.mimeType || 'auto', disabled
),
!this.isConsumer() ? null : fc.getSelect(
new Field({
id: 'contentEncoding',
label: {
en: 'Content Encoding',
},
options: [
{
label: 'utf8',
value: 'utf8',
},
{
label: 'hex',
value: 'hex',
},
{
label: 'ascii',
value: 'ascii',
},
{
label: 'base64',
value: 'base64',
},
{
label: 'utf16le',
value: 'utf16le',
},
{
label: 'latin1',
value: 'latin1',
},
{
label: 'binary',
value: 'binary',
},
],
defaultValue: 'utf8',
}),
model.contentEncoding || 'utf8', disabled
),
/*
!this.isConsumer() ? null : fc.getTextArea(
new Field({
Expand Down