{"type":"doc","content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{"minLevel":{"value":"2"},"maxLevel":{"value":"2"},"type":{"value":"flat"}},"macroMetadata":{"macroId":{"value":"812af86120b7d64463b747ac09afa7ed"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"72c9b4a1-f493-4b84-b268-16358df35c8b"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"Overview","type":"text"}]},{"type":"paragraph","content":[{"text":"Proofpoint Targeted Attack Protection","type":"text","marks":[{"type":"strong"}]},{"text":" (TAP) helps you stay ahead of attackers with an innovative approach that detects, analyzes and blocks advanced threats.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Devo collector features","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"0ab3b2ef-2f2a-4b04-9967-e08c1200cb68"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Feature","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Details","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Allow parallel downloading (","type":"text"},{"text":"multipod","type":"text","marks":[{"type":"code"}]},{"text":")","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"not allowed","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Running environments","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"collector server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"on-premise","type":"text","marks":[{"type":"code"}]}]}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Populated Devo events","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"table","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Flattening preprocessing","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"no","type":"text","marks":[{"type":"code"}]}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Data sources","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"70dfd519-9828-4b6a-a205-54ce73efce97"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Data source","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[280.0]},"content":[{"type":"paragraph","content":[{"text":"API endpoint","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[49.0]},"content":[{"type":"paragraph","content":[{"text":"Collector service name","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[74.0]},"content":[{"type":"paragraph","content":[{"text":"Devo table","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[101.0]},"content":[{"type":"paragraph","content":[{"text":"Available from release","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Blocked clicks","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Fetch events for clicks to malicious URLs blocked in the specified time period","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[280.0]},"content":[{"type":"paragraph","content":[{"text":"/v2/siem/clicks/blocked","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[49.0]},"content":[{"type":"paragraph","content":[{"text":"clicksBlocked","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[74.0]},"content":[{"type":"paragraph","content":[{"text":"mail.proofpoint.tapsiem_v2.clicksblocked","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[101.0]},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"v2.1.1","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Permitted clicks","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Fetch events for clicks to malicious URLs permitted in the specified time period","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[280.0]},"content":[{"type":"paragraph","content":[{"text":"/v2/siem/clicks/permitted","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[49.0]},"content":[{"type":"paragraph","content":[{"text":"clicksPermitted","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[74.0]},"content":[{"type":"paragraph","content":[{"text":"mail.proofpoint.tapsiem_v2.clickspermitted","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[101.0]},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"v2.1.1","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Blocked messages","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Fetch events for messages blocked in the specified time period that contained a known threat.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[280.0]},"content":[{"type":"paragraph","content":[{"text":"/v2/siem/messages/blocked","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[49.0]},"content":[{"type":"paragraph","content":[{"text":"messagesBlocked","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[74.0]},"content":[{"type":"paragraph","content":[{"text":"mail.proofpoint.tapsiem_v2.messagesblocked","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[101.0]},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"v2.1.1","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Delivered message","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Fetch events for messages delivered in the specified time period which contained a known threat.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[280.0]},"content":[{"type":"paragraph","content":[{"text":"/v2/siem/messages/delivered","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[49.0]},"content":[{"type":"paragraph","content":[{"text":"messagesDelivered","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[74.0]},"content":[{"type":"paragraph","content":[{"text":"mail.proofpoint.tapsiem_v2.messagesdelivered","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[101.0]},"content":[{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"v2.1.1","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Threats","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"The Threats API allows administrators to pull detailed attributes about individual threats observed in their environment. It can be used to retrieve more intelligence for threats identified in the SIEM or Campaign API responses","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[280.0]},"content":[{"type":"paragraph","content":[{"text":"siem/all then -> v2/threat/summary/","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[49.0]},"content":[{"type":"paragraph","content":[{"text":"threats","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[74.0]},"content":[{"type":"paragraph","content":[{"text":"mail.proofpoint.tap_threats.events","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[101.0]},"content":[{"type":"paragraph"}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Threats if Forensics is Enabled","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"If Forensics is enabled the events are flattened into the table","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[280.0]},"content":[{"type":"paragraph","content":[{"text":"v2/threat/summary/ then -> v2/threat/summary/&includecampaignforensics=true","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[49.0]},"content":[{"type":"paragraph","content":[{"text":"threats","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[74.0]},"content":[{"type":"paragraph","content":[{"text":"mail.proofpoint.tap_threats.events","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[101.0]},"content":[{"type":"paragraph"}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Campaigns","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"The Campaign API allows administrators to pull campaign IDs in a timeframe and specific details about campaigns, including: their description; the actor, malware family, and techniques associated with the campaign; and the threat variants which have been associated with the campaign","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[280.0]},"content":[{"type":"paragraph","content":[{"text":"v2/campaign/","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[49.0]},"content":[{"type":"paragraph","content":[{"text":"campaigns","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[74.0]},"content":[{"type":"paragraph","content":[{"text":"mail.proofpoint.tap_campaigns.events","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[101.0]},"content":[{"type":"paragraph"}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Campaigns if Forensics is Enabled","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"If Forensics is enabled the events are flattened into the table","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[280.0]},"content":[{"type":"paragraph","content":[{"text":"v2/forensics?campaignId=","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[49.0]},"content":[{"type":"paragraph","content":[{"text":"campaigns","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[74.0]},"content":[{"type":"paragraph","content":[{"text":"mail.proofpoint.tap_campaigns.events","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[101.0]},"content":[{"type":"paragraph"}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"People Top Clicks","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"The People API allows administrators to identify which users in their organizations were most attacked or are the top clickers during a specified period. Fetch the identities and attack index of the top clickers within your organization for a given period. Top clickers are the users who have demonstrated a tendency to click on malicious URLs, regardless of whether the clicks were blocked or not. Knowing who are more susceptible to threats is useful for proactive security approaches such as security training assignments.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[280.0]},"content":[{"type":"paragraph","content":[{"text":"v2/people/top-clickers","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[49.0]},"content":[{"type":"paragraph","content":[{"text":"people_topclicks","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[74.0]},"content":[{"type":"paragraph","content":[{"text":"mail.proofpoint.tap_people.top_clicks","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[101.0]},"content":[{"type":"paragraph"}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"People VAP","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"The People API allows administrators to identify which users in their organizations were most attacked or are the top clickers during a specified period. Fetch the identities and attack index breakdown of Very Attacked People within your organization for a given period.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[280.0]},"content":[{"type":"paragraph","content":[{"text":"v2/people/vap","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[49.0]},"content":[{"type":"paragraph","content":[{"text":"people_vap","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[74.0]},"content":[{"type":"paragraph","content":[{"text":"mail.proofpoint.tap_people.vap","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[101.0]},"content":[{"type":"paragraph"}]}]}]},{"type":"paragraph","content":[{"text":"For more information on how the events are parsed, ","type":"text"},{"text":"visit our page","type":"text","marks":[{"type":"link","attrs":{"href":"https://devodocs.atlassian.net/wiki/spaces/latest/pages/94662170"}}]},{"text":".","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Flattening preprocessing","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"cacaca35-cd2a-4201-b143-a0da6213e14a"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[198.0]},"content":[{"type":"paragraph","content":[{"text":"Data source","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[186.0]},"content":[{"type":"paragraph","content":[{"text":"Collector service","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"Optional","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Flattening details","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[198.0]},"content":[{"type":"paragraph","content":[{"text":"Blocked messages","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[186.0]},"content":[{"type":"paragraph","content":[{"text":"messagesBlocked","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"yes","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"not required","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[198.0]},"content":[{"type":"paragraph","content":[{"text":"Delivered message","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[186.0]},"content":[{"type":"paragraph","content":[{"text":"messagesDelivered","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[126.0]},"content":[{"type":"paragraph","content":[{"text":"yes","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"not required","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Accepted authentication methods","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"20b4d40d-7a91-414b-946a-6bb3298cd735"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[321.0]},"content":[{"type":"paragraph","content":[{"text":"Authentication method","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[215.0]},"content":[{"type":"paragraph","content":[{"text":"username","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[224.0]},"content":[{"type":"paragraph","content":[{"text":"password","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[321.0]},"content":[{"type":"paragraph","content":[{"text":"credentials","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[215.0]},"content":[{"type":"paragraph","content":[{"type":"status","attrs":{"color":"green","style":"bold","text":"REQUIRED","localId":"3a023680-5bd7-485f-95d6-e759bf077217"}}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[224.0]},"content":[{"type":"paragraph","content":[{"type":"status","attrs":{"color":"green","style":"bold","text":"REQUIRED","localId":"f9542841-fe6e-4d76-b6ad-cf1883201682"}}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Minimum configuration required for basic pulling","type":"text"}]},{"type":"paragraph","content":[{"text":"Although this collector supports advanced configuration, the fields required to retrieve data with basic configuration are defined below.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"This minimum configuration refers exclusively to those specific parameters of this integration. There are more required parameters related to the generic behavior of the collector. Check setting sections for details.","type":"text"}]}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"c21a6d21-f9a6-4dc7-b288-612f5ebdd809"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[293.0]},"content":[{"type":"paragraph","content":[{"text":"Setting","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[465.0]},"content":[{"type":"paragraph","content":[{"text":"Details","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[293.0]},"content":[{"type":"paragraph","content":[{"text":"username","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[465.0]},"content":[{"type":"paragraph","content":[{"text":"The username for proofpoint Tap","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[293.0]},"content":[{"type":"paragraph","content":[{"text":"password","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[465.0]},"content":[{"type":"paragraph","content":[{"text":"The password(credential) for proofpoint","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[293.0]},"content":[{"type":"paragraph","content":[{"text":"start_time_in_utc_format","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[465.0]},"content":[{"type":"paragraph","content":[{"text":"Start Time which is not more than 7 days into the past","type":"text"}]}]}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"See the ","type":"text"},{"text":"Accepted authentication methods","type":"text","marks":[{"type":"strong"}]},{"text":" section to verify what settings are required based on the desired authentication method.","type":"text"}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"API Limits, issues ","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"for services ","type":"text"},{"text":"clicksBlocked","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"clicksPermitted","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"messageBlocked","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"messageDelivered ","type":"text","marks":[{"type":"code"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"ClicksBlocked","type":"text","marks":[{"type":"code"}]},{"text":" (","type":"text"},{"text":"/v2/siem/clicks/blocked","type":"text","marks":[{"type":"code"}]},{"text":") ,","type":"text"},{"text":" messageBlocked","type":"text","marks":[{"type":"code"}]},{"text":" (","type":"text"},{"text":"/v2/siem/messages/blocked","type":"text","marks":[{"type":"code"}]},{"text":"), ","type":"text"},{"text":"messageDelivered ","type":"text","marks":[{"type":"code"}]},{"text":"(","type":"text"},{"text":"/v2/siem/messages/delivered","type":"text","marks":[{"type":"code"}]},{"text":") can collectively make 1800 requests per day as per","type":"text"},{"text":" api documentation","type":"text","marks":[{"type":"link","attrs":{"href":"https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/SIEM_API"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"clicksPermitted","type":"text","marks":[{"type":"code"}]},{"text":" (","type":"text"},{"text":"/v2/siem/clicks/permitted","type":"text","marks":[{"type":"code"}]},{"text":") can make 1800 reuqests per day as per ","type":"text"},{"text":"api documentation.","type":"text","marks":[{"type":"link","attrs":{"href":"https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/SIEM_API"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"To simplify the code and handle all the four services, code has been written in a way that each of the four service can make 220 request per day.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"We have put a ","type":"text"},{"text":"limit ","type":"text","marks":[{"type":"strong"}]},{"text":"on the api call,","type":"text"},{"text":" ","type":"text","marks":[{"type":"strong"}]},{"text":"which will block the api requests after 220 requests for a service.","type":"text"},{"text":" ","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"If the collector took, let’s say, 15 hours to make 220 requests, the collector will block the api call for 9 hours (9 + 15 = 24 hours) and it will resume making api requests after 9 hours. For these 9 hours no ingestion will occur for that service.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Issues","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Let’s say all the services make 1000 request in 15 hours and then due to some issue collector restarts. Now, after the restart collector is going to assume that service still make a 1800 request , ignoring the 1000 requests collector made before the restart. This will lead to surpass the 1800 requets per day api limit causing a 429 error. ","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"for services ","type":"text"},{"text":"threats","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"campaigns","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"people_topclicks","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"people_vap ","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph"},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"threats","type":"text","marks":[{"type":"code"}]},{"text":" service has no api limit","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"people_topclicks","type":"text","marks":[{"type":"code"}]},{"text":" (","type":"text"},{"text":"v2/people/top-clickers","type":"text","marks":[{"type":"code"}]},{"text":") and ","type":"text"},{"text":"people_vap","type":"text","marks":[{"type":"code"}]},{"text":" (","type":"text"},{"text":"v2/people/vap","type":"text","marks":[{"type":"code"}]},{"text":") services can make only 50 requests in 1 day as per the ","type":"text"},{"text":"api documentation","type":"text","marks":[{"type":"link","attrs":{"href":"https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/People_API"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"campaigns","type":"text","marks":[{"type":"code"}]},{"text":" (","type":"text"},{"text":"v2/campaign/ids","type":"text","marks":[{"type":"code"}]},{"text":") service can only make 50 request in 1 day as per the ","type":"text"},{"text":"api documentation","type":"text","marks":[{"type":"link","attrs":{"href":"https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/Campaign_API"}}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"We have put a ","type":"text"},{"text":"limit ","type":"text","marks":[{"type":"strong"}]},{"text":"on the api call,","type":"text"},{"text":" ","type":"text","marks":[{"type":"strong"}]},{"text":"which will block the api requests after 50 requests for a service","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"If the collector took , let’s say, 10 hours to make 50 request, the collector will block the api call for 14 hours (10 + 14 = 24 hours) and it will resume making api requests after 14 hours. For these 14 hours no ingestion will occur for that service","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Issues","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Let’s say a service make 30 request in 3 hours and then due to some issue collector restarts. Now, after the restart collector is going to assume that service still make a 50 request , ignoring the 30 requests collector made before the restart. This will lead to surpass the 50 requets per day api limit causing a 429 error for that service. ","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Run the collector","type":"text"}]},{"type":"paragraph","content":[{"text":"Once the data source is configured, you can either send us the required information if you want us to host and manage the collector for you (","type":"text"},{"text":"Cloud collector","type":"text","marks":[{"type":"underline"}]},{"text":"), or deploy and host the collector in your own machine using a Docker image (","type":"text"},{"text":"On-premise collector","type":"text","marks":[{"type":"underline"}]},{"text":").","type":"text"}]},{"type":"bodiedExtension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"rw-ui-tabs-macro","parameters":{"macroParams":{},"macroMetadata":{"macroId":{"value":"6a805bc0-abf6-4d98-987c-6a9164c77642"},"schemaVersion":{"value":"1"},"placeholder":[{"type":"icon","data":{"url":"https://static.dg.refinedtheme.refinedwiki.com/refinedtheme-for-cloud/macro-icons/tab-container.png"}}],"title":"Refined Tab Container"}},"localId":"51a38310-b288-4fb7-a20e-43dcd020e55a"},"content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"rw-tab","parameters":{"macroParams":{"title":{"value":"Cloud collector"}},"macroMetadata":{"macroId":{"value":"2825992cb366f7c43cdf1373692654dc"},"schemaVersion":{"value":"1"},"placeholder":[{"type":"icon","data":{"url":"https://static.dg.refinedtheme.refinedwiki.com/refinedtheme-for-cloud/macro-icons/tab.png"}}],"title":"Refined Tab"}},"localId":"0f26672c-3765-45d9-a825-cd6014850d1a"}},{"type":"paragraph","content":[{"text":"We use a piece of software called Collector Server to host and manage all our available collectors. If you want us to host this collector for you, ","type":"text"},{"text":"get in touch with us","type":"text","marks":[{"type":"link","attrs":{"href":"https://devo.my.site.com/support/login?ec=302&startURL=%2Fsupport%2Fs%2F"}}]},{"text":" and we will guide you through the configuration.","type":"text"},{"type":"hardBreak"},{"text":"Editing the json file","type":"text","marks":[{"type":"strong"}]}]},{"type":"codeBlock","content":[{"text":"{\n \"global_overrides\": {\n \"debug\": false\n },\n \"inputs\": {\n \"proofpoint_tap\": {\n \"id\": \"\",\n \"enabled\": true,\n \"credentials\": {\n \"username\": \"\",\n \"password\": \"\"\n },\n \"services\": {\n \"clicksBlocked\": {\n \"start_time_in_utc_format\": \"\"\n },\n \"clicksPermitted\": {\n \"start_time_in_utc_format\": \"\"\n },\n \"messagesBlocked\": {\n \"start_time_in_utc_format\": \"\"\n },\n \"messagesDelivered\": {\n \"start_time_in_utc_format\": \"\"\n },\n \"campaigns\": {\n \"start_time_in_utc_format\": \"\",\n \"forensics\": true\n },\n \"threats\": {\n \"start_time_in_utc_format\": \"\",\n \"forensics\": true,\n \"include_campaign_forensics\": true\n },\n \"people_topclicks\": {\n \"start_time_in_utc_format\": \"\"\n },\n \"people_vap\": {\n \"start_time_in_utc_format\": \"\"\n }\n }\n }\n }\n}","type":"text"}]},{"type":"paragraph"},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"afc6ad40-d19e-4ed5-8aeb-cb6477f627a5"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"Parameter","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"Data type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"Value range / Format","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"Details","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"int","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Mandatory","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"Minimum Lenght 5","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"Use this param to give a unique id to this input service.","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"This parameter is used to build the persistence address, do not use the same value for multiple collectors. It could cause a collision.","type":"text"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"bool","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Mandatory","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"false","type":"text","marks":[{"type":"code"}]},{"text":" / ","type":"text"},{"text":"true","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"Use this param to enable or disable the given input logic when running the collector. If the value is ","type":"text"},{"text":"true","type":"text","marks":[{"type":"code"}]},{"text":", the input will be run. If the value is ","type":"text"},{"text":"false","type":"text","marks":[{"type":"code"}]},{"text":", it will be ignored.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"str","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Mandatory","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"Minimum Length 1","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"Provide the value of username used for authentication","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"str","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Mandatory","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"Minimum Length 1","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"Provide the value of password used for authentication","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"str","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Mandatory","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"Minimum Length 1","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":"%Y-%m-%dT%H:%M:%SZ","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"This configuration allows you to set a custom date as the beginning of the period to download. This allows the events but it can’t be more than 7 days in the past.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"request_period_in_seconds","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"int","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Optional","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"Minimum Length 1","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"This configuration allows us to set the “seconds” after which next pull cycle to be made. Allowed limit of number of api requests is 1800 per day so we have kept 600 seconds as a default value.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"str","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Optional","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"A devo tag","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"This parameter allows to define a custom devo tag.","type":"text"}]}]}]}]},{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"rw-tab","parameters":{"macroParams":{"title":{"value":"On-premise collector"}},"macroMetadata":{"macroId":{"value":"0e08d10a084320745cfcad174c481cfb"},"schemaVersion":{"value":"1"},"placeholder":[{"type":"icon","data":{"url":"https://static.dg.refinedtheme.refinedwiki.com/refinedtheme-for-cloud/macro-icons/tab.png"}}],"title":"Refined Tab"}},"localId":"3392ad6b-e017-4587-96ac-e5e9d4459cb2"}},{"type":"paragraph","content":[{"text":"This data collector can be run in any machine that has the Docker service available because it should be executed as a docker container. The following sections explain how to prepare all the required setup for having the data collector running.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Structure","type":"text"}]},{"type":"paragraph","content":[{"text":"The following directory structure should be created for being used when running the collector:","type":"text"}]},{"type":"codeBlock","content":[{"text":"\n└── devo-collectors/\n └── /\n ├── certs/\n │ ├── chain.crt\n │ ├── .key\n │ └── .crt\n ├── state/\n └── config/ \n └── config.yaml ","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Replace ","type":"text"},{"text":"","type":"text","marks":[{"type":"code"}]},{"text":" with the proper value.","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Devo credentials","type":"text"}]},{"type":"paragraph","content":[{"text":"In Devo, go to ","type":"text"},{"text":"Administration → Credentials → X.509 Certificates","type":"text","marks":[{"type":"strong"}]},{"text":", download the ","type":"text"},{"text":"Certificate","type":"text","marks":[{"type":"strong"}]},{"text":", ","type":"text"},{"text":"Private key","type":"text","marks":[{"type":"strong"}]},{"text":" and ","type":"text"},{"text":"Chain CA","type":"text","marks":[{"type":"strong"}]},{"text":" and save them in ","type":"text"},{"text":"/certs/","type":"text","marks":[{"type":"code"}]},{"text":". Learn more about security credentials in Devo ","type":"text"},{"text":"here","type":"text","marks":[{"type":"link","attrs":{"href":"#"}}]},{"text":".","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":726,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":950,"alt":"image-20240108-081455.png","id":"5fbd3cf3-e9a5-4313-b357-e146e99289ba","collection":"contentId-457637902","type":"file","height":448}}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Replace ","type":"text"},{"text":"","type":"text","marks":[{"type":"code"}]},{"text":" with the proper value.","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Editing the config.yaml file","type":"text"}]},{"type":"codeBlock","content":[{"text":"globals:\n debug: false\n id: \n name: \n persistence:\n type: filesystem\n config:\n directory_name: state\noutputs:\n devo_us_1:\n type: devo_platform\n config:\n address: \n port: 443\n type: SSL\n chain: \n cert: \n key: \ninputs:\n proofpoint_tap:\n id: \n enabled: true\n credentials:\n username: \n password: \n services:\n clicksBlocked:\n start_time_in_utc_format: # example 2022-05-14T00:00:00Z\n clicksPermitted:\n start_time_in_utc_format: # example 2022-05-14T00:00:00Z\n messagesBlocked:\n start_time_in_utc_format: # example 2022-05-14T00:00:00Z\n messagesDelivered:\n start_time_in_utc_format: # example 2022-05-14T00:00:00Z\n threats:\n start_time_in_utc_format: # example 2022-05-14T00:00:00Z\n forensics: true\n include_campaign_forensics: true\n campaigns:\n start_time_in_utc_format: # example 2022-05-14T00:00:00Z\n forensics: true\n people_topclicks:\n start_time_in_utc_format: # example 2022-05-14T00:00:00Z\n people_vap:\n start_time_in_utc_format: # example 2022-05-14T00:00:00Z","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"All defined service entities will be executed by the collector. If you do not want to run any of them, just remove the entity from the ","type":"text"},{"text":"services","type":"text","marks":[{"type":"code"}]},{"text":" object.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"Replace the placeholders with your required values following the description table below:","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"10b8e101-8cd1-4310-af0d-d552f78debb7"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"Parameter","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"Data type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"Value range / Format","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"Details","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"int","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Mandatory","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"Minimum Lenght 5","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"Use this param to give a unique id to this input service.","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"This parameter is used to build the persistence address, do not use the same value for multiple collectors. It could cause a collision.","type":"text"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"bool","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Mandatory","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"false","type":"text","marks":[{"type":"code"}]},{"text":" / ","type":"text"},{"text":"true","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"Use this param to enable or disable the given input logic when running the collector. If the value is ","type":"text"},{"text":"true","type":"text","marks":[{"type":"code"}]},{"text":", the input will be run. If the value is ","type":"text"},{"text":"false","type":"text","marks":[{"type":"code"}]},{"text":", it will be ignored.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"str","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Mandatory","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"Minimum Length 1","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"Provide the value of username used for authentication","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"str","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Mandatory","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"Minimum Length 1","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"Provide the value of password used for authentication","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"str","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Mandatory","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"Minimum Length 1","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":"%Y-%m-%dT%H:%M:%SZ","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"This configuration allows you to set a custom date as the beginning of the period to download. This allows the events but it can’t be more than 7 days in the past.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"request_period_in_seconds","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"int","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Optional","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"Minimum Length 1","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"This configuration allows us to set the “seconds” after which next pull cycle to be made. Allowed limit of number of api requests is 1800 per day so we have kept 600 seconds as a default value.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[168.0]},"content":[{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[107.0]},"content":[{"type":"paragraph","content":[{"text":"str","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[111.0]},"content":[{"type":"paragraph","content":[{"text":"Optional","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[183.0]},"content":[{"type":"paragraph","content":[{"text":"A devo tag","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[157.0]},"content":[{"type":"paragraph","content":[{"text":"This parameter allows to define a custom devo tag.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Download the Docker image","type":"text"}]},{"type":"paragraph","content":[{"text":"The collector should be deployed as a Docker container. Download the Docker image of the collector as a .tgz file by clicking the link in the following table:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"94785be3-54fc-4266-9d73-deee265f1451"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[363.0]},"content":[{"type":"paragraph","content":[{"text":"Collector Docker image","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[363.0]},"content":[{"type":"paragraph","content":[{"text":"SHA-256 hash","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[363.0]},"content":[{"type":"paragraph","content":[{"text":"collector-proofpoint_tap_if-docker-image-3.2.0","type":"text","marks":[{"type":"link","attrs":{"href":"https://drive.google.com/file/d/1aHKDhQc79NrMbrdKj9XX7nOK2T7TFlRf/view?usp=drive_link"}}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[363.0]},"content":[{"type":"paragraph","content":[{"text":"9c20f48b3bf1c625b0097dbaaed544d0978cb455b88fc24370d72d45ac4e2fcc","type":"text","marks":[{"type":"code"}]}]}]}]}]},{"type":"paragraph","content":[{"text":"Use the following command to add the Docker image to the system:","type":"text"}]},{"type":"codeBlock","content":[{"text":"gunzip -c -.tgz | docker load","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Once the Docker image is imported, it will show the real name of the Docker image (including version info). Replace ","type":"text"},{"text":"","type":"text","marks":[{"type":"code"}]},{"text":" and ","type":"text"},{"text":"","type":"text","marks":[{"type":"code"}]},{"text":" with a proper value.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"The Docker image can be deployed on the following services:","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Docker","type":"text"}]},{"type":"paragraph","content":[{"text":"Execute the following command on the root directory ","type":"text"},{"text":"/devo-collectors//","type":"text","marks":[{"type":"code"}]}]},{"type":"codeBlock","content":[{"text":"docker run \n--name collector- \n--volume $PWD/certs:/devo-collector/certs \n--volume $PWD/config:/devo-collector/config \n--volume $PWD/state:/devo-collector/state \n--env CONFIG_FILE=config.yaml \n--rm \n--interactive \n--tty \n:","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Replace ","type":"text"},{"text":"","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"","type":"text","marks":[{"type":"code"}]},{"text":" and ","type":"text"},{"text":"","type":"text","marks":[{"type":"code"}]},{"text":" with the proper values.","type":"text"}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Docker Compose","type":"text"}]},{"type":"paragraph","content":[{"text":"The following Docker Compose file can be used to execute the Docker container. It must be created in the ","type":"text"},{"text":"/devo-collectors//","type":"text","marks":[{"type":"code"}]},{"text":" directory.","type":"text"}]},{"type":"codeBlock","content":[{"text":"version: '3'\nservices:\n collector-:\n image: :${IMAGE_VERSION:-latest}\n container_name: collector-\n volumes:\n - ./certs:/devo-collector/certs\n - ./config:/devo-collector/config\n - ./credentials:/devo-collector/credentials\n - ./state:/devo-collector/state\n environment:\n - CONFIG_FILE=${CONFIG_FILE:-config.yaml}","type":"text"}]},{"type":"paragraph","content":[{"text":"To run the container using docker-compose, execute the following command from the ","type":"text"},{"text":"/devo-collectors//","type":"text","marks":[{"type":"code"}]},{"text":" directory:","type":"text"}]},{"type":"codeBlock","content":[{"text":"IMAGE_VERSION= docker-compose up -d","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Replace ","type":"text"},{"text":"","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"","type":"text","marks":[{"type":"code"}]},{"text":" and ","type":"text"},{"text":"","type":"text","marks":[{"type":"code"}]},{"text":" with the proper values.","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Collector services detail","type":"text"}]},{"type":"paragraph","content":[{"text":"This section is intended to explain how to proceed with specific actions for services.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Messages blocked","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"No. of request this service can make in a day is 220.","type":"text"}]}]},{"type":"expand","attrs":{"title":"Verify data collection "},"content":[{"type":"paragraph","content":[{"text":"Once the collector has been launched, it is important to check if the ingestion is performed in a proper way. To do so, go to the collector’s logs console. ","type":"text"}]},{"type":"paragraph","content":[{"text":"This service has the following components:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"0fc14022-ade5-4eae-ba9b-eaa9749ea4a9"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-neutral, #F4F5F7)","rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Component","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-neutral, #F4F5F7)","rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Setup","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of authenticating the service and managing the token expiration when needed.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Puller","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of pulling the data in a organized way and delivering the events via SDK.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Setup output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful run has the following output messages for the setup module:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-26T09:05:33.934 INFO OutputProcess::MainThread -> DevoSender(lookup_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:05:33.935 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(lookup_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:05:33.935 INFO OutputProcess::MainThread -> DevoSenderManager(lookup_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:05:33.935 INFO OutputProcess::MainThread -> DevoSender(internal_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:05:33.935 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(internal_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:05:33.935 INFO OutputProcess::DevoSenderManager(lookup_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(lookup_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:05:33.935 INFO OutputProcess::MainThread -> DevoSenderManager(internal_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:05:33.935 INFO InputProcess::MainThread -> Validating defined module definition\n2023-12-26T09:05:33.936 INFO OutputProcess::DevoSenderManager(internal_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(internal_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:05:33.936 INFO OutputProcess::OutputInternalConsumer(internal_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputInternalConsumer(internal_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:05:33.936 INFO OutputProcess::OutputLookupConsumer(lookup_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputLookupConsumer(lookup_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:05:33.939 INFO InputProcess::MainThread -> Validating common input config\n2023-12-26T09:05:33.940 INFO InputProcess::MainThread -> Validating service input config\n2023-12-26T09:05:33.940 INFO InputProcess::MainThread -> Running overriding rules\n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> Validating the rate limiter config given by the user\n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> setting has not been defined. The generic settings will be used instead.\n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> Adding raw config to the collector store\n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> Running custom validation rules\n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> Creating API client.\n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> Created request client: \n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) Finalizing the execution of init_variables()\n2023-12-26T09:05:33.942 INFO InputProcess::MainThread -> InputThread(proofpoint_tap,123456) - Starting thread (execution_period=60s)\n2023-12-26T09:05:33.942 INFO InputProcess::MainThread -> ServiceThread(proofpoint_tap,123456,messagesBlocked,predefined) - Starting thread (execution_period=60s)\n2023-12-26T09:05:33.942 INFO InputProcess::MainThread -> ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesBlocked#predefined) -> Starting thread\n2023-12-26T09:05:33.942 WARNING InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesBlocked#predefined) -> The token/header/authentication has not been created yet\n2023-12-26T09:05:33.943 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) - Starting thread\n2023-12-26T09:05:33.943 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> Waiting until setup will be executed\n2023-12-26T09:05:33.947 INFO OutputProcess::MainThread -> [GC] global: 28.2% -> 28.2%, process: RSS(40.77MiB -> 41.64MiB), VMS(926.00MiB -> 926.00MiB)\n2023-12-26T09:05:33.954 INFO InputProcess::MainThread -> [GC] global: 28.2% -> 28.2%, process: RSS(40.29MiB -> 40.29MiB), VMS(421.98MiB -> 421.98MiB)\n2023-12-26T09:05:34.739 INFO OutputProcess::DevoSender(internal_senders,devo_sender_0) -> Created a sender: {\"name\": \"DevoSender(internal_senders,devo_sender_0)\", \"url\": \"collector-eu.devo.io:443\", \"chain_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/chain.crt\", \"cert_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.crt\", \"key_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.key\", \"transport_layer_type\": \"SSL\", \"last_usage_timestamp\": null, \"socket_status\": null}, hostname: \"2023-apac-0046\", session_id: \"140140610316704\"\n2023-12-26T09:05:36.819 INFO InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesBlocked#predefined) -> Setup for module has been successfully executed","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Puller output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful initial run has the following output messages for the puller module:","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note that the ","type":"text"},{"text":"PrePull","type":"text","marks":[{"type":"code"}]},{"text":" action is executed only one time before the first run of the ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"023-12-26T09:05:36.953 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> Pull Started\n2023-12-26T09:05:36.956 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> Time Window FROM: 2023-12-10 11:00:00+00:00 TO: 2023-12-10 12:00:00+00:00\n2023-12-26T09:05:36.956 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> Start_Time_in_utc must be at most 7.00d into the past, Changing the time startTime to be in the specified time\n2023-12-26T09:05:39.320 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561736946):Number of requests made: 1; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-26T09:05:39.321 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> Time Window FROM: 2023-12-19 04:35:36.946633+00:00 TO: 2023-12-19 05:35:36.946633+00:00\n2023-12-26T09:05:41.780 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561736946):Number of requests made: 2; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-26T09:05:41.781 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> Time Window FROM: 2023-12-19 05:35:36.946633+00:00 TO: 2023-12-19 06:35:36.946633+00:00\n2023-12-26T09:05:44.135 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561736946):Number of requests made: 3; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-26T09:05:44.136 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> Time Window FROM: 2023-12-19 06:35:36.946633+00:00 TO: 2023-12-19 07:35:36.946633+00:00\n2023-12-26T09:05:46.495 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561736946):Number of requests made: 4; Number of events received: 1; Number of duplicated events filtered out: 0; Number of events generated and sent: 1; Average of events per second: 0.105.","type":"text"}]},{"type":"paragraph","content":[{"text":"After a successful collector’s execution (that is, no error logs found), you will see the following log message:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-26T09:05:46.495 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561736946):Number of requests made: 4; Number of events received: 1; Number of duplicated events filtered out: 0; Number of events generated and sent: 1; Average of events per second: 0.105.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"The value ","type":"text"},{"text":"@devo_pulling_id","type":"text","marks":[{"type":"code"}]},{"text":" is injected in each event to group all events ingested by the same pull action. You can use it to get the exact events downloaded in that ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action in Devo’s search window.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Restart the persistence"},"content":[{"type":"paragraph","content":[{"text":"This collector uses persistent storage to download events in an orderly fashion and avoid duplicates. In case you want to re-ingest historical data or recreate the persistence, you can restart the persistence of this collector by following these steps:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Edit the configuration file.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Change the value of the ","type":"text"},{"text":"start_time_in_utc_format","type":"text","marks":[{"type":"code"}]},{"text":" parameter to a different one.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Save the changes.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Restart the collector.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"The collector will detect this change and will restart the persistence using the parameters of the configuration file or the default configuration in case it has not been provided.","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Note that this action clears the persistence and cannot be recovered in any way. Resetting persistence could result in duplicate or lost events.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Troubleshooting"},"content":[{"type":"paragraph","content":[{"text":"This collector has different security layers that detect both an invalid configuration and abnormal operation. This table will help you detect and resolve the most common errors.","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"c5ca1ac5-aea2-4e1c-851b-a87aa0627c5a"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Error type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error ID","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error message","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Cause","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Solution","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"SetupError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"100","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"username and password is not correct","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure that credentials are correct.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"101","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error occurred while retrieving events from ProofPoint server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"start_time_in_utc is in future or not in proper format","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure the start time is not in future and not in proper format","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"PullError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"300","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server : {summery} , {details}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"This error happens when the collector tries to fetch the data from API.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"In this error you will find the HTTP error code as well as the summary and details.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":" ","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"301","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some Error occurred while retrieving events from ProofPoint server : {Exception}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some exception occured while making the API request.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Reach out to the developer with the exact error message.","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Messages delivered","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"No. of request this service can make in a day is 220.","type":"text"}]}]},{"type":"expand","attrs":{"title":"Verify data collection"},"content":[{"type":"paragraph","content":[{"text":"Once the collector has been launched, it is important to check if the ingestion is performed in a proper way. To do so, go to the collector’s logs console.","type":"text"}]},{"type":"paragraph","content":[{"text":"This service has the following components:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"89425a52-17b9-4fa6-ac73-873835516241"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Component","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Setup","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of authenticating the service and managing the token expiration when needed.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Puller","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of pulling the data in a organized way and delivering the events via SDK.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Setup output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful run has the following output messages for the setup module:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-26T09:15:01.181 INFO OutputProcess::MainThread -> Process started\n2023-12-26T09:15:01.182 INFO MainProcess::MainThread -> Started all object from \"MainProcess\" process\n2023-12-26T09:15:01.182 INFO InputProcess::MainThread -> Process Started\n2023-12-26T09:15:01.204 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) Starting the execution of init_variables()\n2023-12-26T09:15:01.204 INFO InputProcess::MainThread -> Validating service metadata\n2023-12-26T09:15:01.206 INFO InputProcess::MainThread -> Validating defined module definition\n2023-12-26T09:15:01.207 INFO OutputProcess::MainThread -> DevoSender(standard_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(standard_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManager(standard_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSender(lookup_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(lookup_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManager(lookup_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputStandardConsumer(standard_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputStandardConsumer(standard_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO InputProcess::MainThread -> Validating common input config\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(standard_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(standard_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(lookup_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(lookup_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSender(internal_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputLookupConsumer(lookup_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputLookupConsumer(lookup_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(internal_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSenderManager(internal_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(internal_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(internal_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputInternalConsumer(internal_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputInternalConsumer(internal_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Validating service input config\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Running overriding rules\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Validating the rate limiter config given by the user\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> setting has not been defined. The generic settings will be used instead.\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Adding raw config to the collector store\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Running custom validation rules\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Creating API client.\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Created request client: \n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) Finalizing the execution of init_variables()\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> InputThread(proofpoint_tap,123456) - Starting thread (execution_period=60s)\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ServiceThread(proofpoint_tap,123456,messagesDelivered,predefined) - Starting thread (execution_period=60s)\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> Starting thread\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) - Starting thread\n2023-12-26T09:15:01.212 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> Waiting until setup will be executed\n2023-12-26T09:15:01.212 WARNING InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> The token/header/authentication has not been created yet\n2023-12-26T09:15:01.221 INFO OutputProcess::MainThread -> [GC] global: 28.0% -> 28.1%, process: RSS(40.77MiB -> 41.89MiB), VMS(926.00MiB -> 926.00MiB)\n2023-12-26T09:15:01.225 INFO InputProcess::MainThread -> [GC] global: 28.0% -> 28.1%, process: RSS(40.29MiB -> 40.29MiB), VMS(421.98MiB -> 421.98MiB)\n2023-12-26T09:15:01.713 INFO OutputProcess::DevoSender(internal_senders,devo_sender_0) -> Created a sender: {\"name\": \"DevoSender(internal_senders,devo_sender_0)\", \"url\": \"collector-eu.devo.io:443\", \"chain_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/chain.crt\", \"cert_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.crt\", \"key_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.key\", \"transport_layer_type\": \"SSL\", \"last_usage_timestamp\": null, \"socket_status\": null}, hostname: \"2023-apac-0046\", session_id: \"139871239825152\"\n2023-12-26T09:15:03.324 INFO InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> Setup for module has been successfully executed","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Puller output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful initial run has the following output messages for the puller module:","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note that the ","type":"text"},{"text":"PrePull","type":"text","marks":[{"type":"code"}]},{"text":" action is executed only one time before the first run of the ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"2023-12-26T09:15:04.224 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> Pull Started\n2023-12-26T09:15:04.227 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> Time Window FROM: 2023-12-25 03:00:00+00:00 TO: 2023-12-25 04:00:00+00:00\n2023-12-26T09:15:12.140 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703562304216):Number of requests made: 1; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-26T09:15:12.140 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> Time Window FROM: 2023-12-25 04:00:00+00:00 TO: 2023-12-25 05:00:00+00:00\n2023-12-26T09:15:13.985 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703562304216):Number of requests made: 2; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-26T09:15:13.986 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> Time Window FROM: 2023-12-25 05:00:00+00:00 TO: 2023-12-25 06:00:00+00:00\n2023-12-26T09:15:16.077 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703562304216):Number of requests made: 3; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-26T09:15:16.077 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> Time Window FROM: 2023-12-25 06:00:00+00:00 TO: 2023-12-25 07:00:00+00:00\n2023-12-26T09:15:18.287 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703562304216):Number of requests made: 4; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.","type":"text"}]},{"type":"paragraph","content":[{"text":"After a successful collector’s execution (that is, no error logs found), you will see the following log message:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-22T10:28:58.153 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksBlocked,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703220842731):Number of requests made: 164; Number of events received: 14; Number of duplicated events filtered out: 0; Number of events generated and sent: 14; Average of events per second: 0.047.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"The value ","type":"text"},{"text":"@devo_pulling_id","type":"text","marks":[{"type":"code"}]},{"text":" is injected in each event to group all events ingested by the same pull action. You can use it to get the exact events downloaded in that ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action in Devo’s search window.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Restart the persistence"},"content":[{"type":"paragraph","content":[{"text":"This collector uses persistent storage to download events in an orderly fashion and avoid duplicates. In case you want to re-ingest historical data or recreate the persistence, you can restart the persistence of this collector by following these steps:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Edit the configuration file.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Change the value of the ","type":"text"},{"text":"start_time_in_utc_format","type":"text","marks":[{"type":"code"}]},{"text":" parameter to a different one.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Save the changes.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Restart the collector.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"The collector will detect this change and will restart the persistence using the parameters of the configuration file or the default configuration in case it has not been provided.","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Note that this action clears the persistence and cannot be recovered in any way. Resetting persistence could result in duplicate or lost events.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Troubleshooting"},"content":[{"type":"paragraph","content":[{"text":"This collector has different security layers that detect both an invalid configuration and abnormal operation. This table will help you detect and resolve the most common errors.","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"76e31a26-9bc6-48ea-9afb-413b158dd415"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Error type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error ID","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error message","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Cause","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Solution","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"SetupError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"100","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"username and password is not correct","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure that credentials are correct.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"101","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error occurred while retrieving events from ProofPoint server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"start_time_in_utc is in future or not in proper format","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure the start time is not in future and not in proper format","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"PullError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"300","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server : {summery} , {details}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"This error happens when the collector tries to fetch the data from API.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"In this error you will find the HTTP error code as well as the summary and details.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":" ","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"301","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some Error occurred while retrieving events from ProofPoint server : {Exception}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some exception occured while making the API request.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Reach out to the developer with the exact error message.","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Clicks permitted","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"No. of request this service can make in a day is 220.","type":"text"}]}]},{"type":"expand","attrs":{"title":"Verify data collection"},"content":[{"type":"paragraph","content":[{"text":"Once the collector has been launched, it is important to check if the ingestion is performed in a proper way. To do so, go to the collector’s logs console.","type":"text"}]},{"type":"paragraph","content":[{"text":"This service has the following components:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"579d13f9-98b6-436c-b638-53c8ddbf3c60"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Component","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Setup","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of authenticating the service and managing the token expiration when needed.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Puller","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of pulling the data in a organized way and delivering the events via SDK.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Setup output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful run has the following output messages for the setup module:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-26T08:57:18.936 INFO MainProcess::MainThread -> Started all object from \"MainProcess\" process\n2023-12-26T08:57:18.957 WARNING InputProcess::MainThread -> A previous rate limiter with same \"period_in_seconds\" and \"number_of requests\" was already existing: \"86400/1800\"\n2023-12-26T08:57:18.957 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) Starting the execution of init_variables()\n2023-12-26T08:57:18.957 INFO InputProcess::MainThread -> Validating service metadata\n2023-12-26T08:57:18.959 INFO InputProcess::MainThread -> Validating defined module definition\n2023-12-26T08:57:18.962 INFO InputProcess::MainThread -> Validating common input config\n2023-12-26T08:57:18.963 INFO OutputProcess::MainThread -> DevoSender(standard_senders,devo_sender_0) -> Starting thread\n2023-12-26T08:57:18.963 INFO InputProcess::MainThread -> Validating service input config\n2023-12-26T08:57:18.963 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(standard_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T08:57:18.963 INFO OutputProcess::MainThread -> DevoSenderManager(standard_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T08:57:18.963 INFO OutputProcess::MainThread -> DevoSender(lookup_senders,devo_sender_0) -> Starting thread\n2023-12-26T08:57:18.963 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(lookup_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T08:57:18.963 INFO OutputProcess::OutputStandardConsumer(standard_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputStandardConsumer(standard_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T08:57:18.963 INFO OutputProcess::MainThread -> DevoSenderManager(lookup_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T08:57:18.963 INFO OutputProcess::DevoSenderManager(standard_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(standard_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T08:57:18.963 INFO InputProcess::MainThread -> Running overriding rules\n2023-12-26T08:57:18.964 INFO InputProcess::MainThread -> Validating the rate limiter config given by the user\n2023-12-26T08:57:18.964 INFO OutputProcess::MainThread -> DevoSender(internal_senders,devo_sender_0) -> Starting thread\n2023-12-26T08:57:18.964 INFO InputProcess::MainThread -> setting has not been defined. The generic settings will be used instead.\n2023-12-26T08:57:18.964 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(internal_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T08:57:18.964 INFO InputProcess::MainThread -> Adding raw config to the collector store\n2023-12-26T08:57:18.964 INFO InputProcess::MainThread -> Running custom validation rules\n2023-12-26T08:57:18.964 INFO OutputProcess::OutputLookupConsumer(lookup_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputLookupConsumer(lookup_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T08:57:18.964 INFO InputProcess::MainThread -> Creating API client.\n2023-12-26T08:57:18.964 INFO InputProcess::MainThread -> Created request client: \n2023-12-26T08:57:18.964 INFO OutputProcess::DevoSenderManager(lookup_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(lookup_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T08:57:18.964 INFO OutputProcess::MainThread -> DevoSenderManager(internal_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T08:57:18.964 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) Finalizing the execution of init_variables()\n2023-12-26T08:57:18.964 INFO OutputProcess::DevoSenderManager(internal_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(internal_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T08:57:18.964 INFO OutputProcess::OutputInternalConsumer(internal_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputInternalConsumer(internal_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T08:57:18.965 INFO InputProcess::MainThread -> InputThread(proofpoint_tap,123456) - Starting thread (execution_period=60s)\n2023-12-26T08:57:18.965 INFO InputProcess::MainThread -> ServiceThread(proofpoint_tap,123456,clicksPermitted,predefined) - Starting thread (execution_period=60s)\n2023-12-26T08:57:18.965 INFO InputProcess::MainThread -> ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,clicksPermitted#predefined) -> Starting thread\n2023-12-26T08:57:18.965 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) - Starting thread\n2023-12-26T08:57:18.965 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> Waiting until setup will be executed\n2023-12-26T08:57:18.965 WARNING InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,clicksPermitted#predefined) -> The token/header/authentication has not been created yet\n2023-12-26T08:57:18.978 INFO InputProcess::MainThread -> [GC] global: 28.6% -> 28.7%, process: RSS(40.04MiB -> 40.04MiB), VMS(421.98MiB -> 421.98MiB)\n2023-12-26T08:57:18.979 INFO OutputProcess::MainThread -> [GC] global: 28.6% -> 28.7%, process: RSS(40.52MiB -> 41.39MiB), VMS(926.00MiB -> 926.00MiB)\n2023-12-26T08:57:19.718 INFO OutputProcess::DevoSender(internal_senders,devo_sender_0) -> Created a sender: {\"name\": \"DevoSender(internal_senders,devo_sender_0)\", \"url\": \"collector-eu.devo.io:443\", \"chain_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/chain.crt\", \"cert_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.crt\", \"key_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.key\", \"transport_layer_type\": \"SSL\", \"last_usage_timestamp\": null, \"socket_status\": null}, hostname: \"2023-apac-0046\", session_id: \"139832919561120\"\n2023-12-26T08:57:22.226 INFO InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,clicksPermitted#predefined) -> Setup for module has been successfully executed","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Puller output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful initial run has the following output messages for the puller module:","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note that the ","type":"text"},{"text":"PrePull","type":"text","marks":[{"type":"code"}]},{"text":" action is executed only one time before the first run of the ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"2023-12-26T08:57:22.974 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> Pull Started\n2023-12-26T08:57:22.980 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> Time Window FROM: 2023-12-21 13:48:14+00:00 TO: 2023-12-21 14:48:14+00:00\n2023-12-26T08:57:25.954 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561242970):Number of requests made: 1; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-26T08:57:25.955 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> Time Window FROM: 2023-12-21 14:48:14+00:00 TO: 2023-12-21 15:48:14+00:00\n2023-12-26T08:57:27.948 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561242970):Number of requests made: 2; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-26T08:57:27.949 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> Time Window FROM: 2023-12-21 15:48:14+00:00 TO: 2023-12-21 16:48:14+00:00\n2023-12-26T08:57:29.784 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561242970):Number of requests made: 3; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-26T08:57:29.785 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> Time Window FROM: 2023-12-21 16:48:14+00:00 TO: 2023-12-21 17:48:14+00:00\n2023-12-26T08:57:31.515 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561242970):Number of requests made: 4; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-26T08:57:31.516 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> Time Window FROM: 2023-12-21 17:48:14+00:00 TO: 2023-12-21 18:48:14+00:00\n2023-12-26T08:57:33.312 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561242970):Number of requests made: 5; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-26T08:57:33.313 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> Time Window FROM: 2023-12-21 18:48:14+00:00 TO: 2023-12-21 19:48:14+00:00\n2023-12-26T08:57:42.836 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561242970):Number of requests made: 6; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.","type":"text"}]},{"type":"paragraph","content":[{"text":"After a successful collector’s execution (that is, no error logs found), you will see the following log message:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-26T08:57:42.836 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561242970):Number of requests made: 6; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"The value ","type":"text"},{"text":"@devo_pulling_id","type":"text","marks":[{"type":"code"}]},{"text":" is injected in each event to group all events ingested by the same pull action. You can use it to get the exact events downloaded in that ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action in Devo’s search window.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Restart the persistence"},"content":[{"type":"paragraph","content":[{"text":"This collector uses persistent storage to download events in an orderly fashion and avoid duplicates. In case you want to re-ingest historical data or recreate the persistence, you can restart the persistence of this collector by following these steps:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Edit the configuration file.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Change the value of the ","type":"text"},{"text":"start_time_in_utc_format","type":"text","marks":[{"type":"code"}]},{"text":" parameter to a different one.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Save the changes.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Restart the collector.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"The collector will detect this change and will restart the persistence using the parameters of the configuration file or the default configuration in case it has not been provided.","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Note that this action clears the persistence and cannot be recovered in any way. Resetting persistence could result in duplicate or lost events.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Troubleshooting"},"content":[{"type":"paragraph","content":[{"text":"This collector has different security layers that detect both an invalid configuration and abnormal operation. This table will help you detect and resolve the most common errors.","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"637b0a67-6ff8-4f33-a797-b91f81491183"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Error type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error ID","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error message","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Cause","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Solution","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"SetupError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"100","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"username and password is not correct","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure that credentials are correct.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"101","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error occurred while retrieving events from ProofPoint server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"start_time_in_utc is in future or not in proper format","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure the start time is not in future and not in proper format","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"PullError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"300","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server : {summery} , {details}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"This error happens when the collector tries to fetch the data from API.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"In this error you will find the HTTP error code as well as the summary and details.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":" ","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"301","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some Error occurred while retrieving events from ProofPoint server : {Exception}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some exception occured while making the API request.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Reach out to the developer with the exact error message.","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Clicks blocked","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"No. of request this service can make in a day is 220.","type":"text"}]}]},{"type":"expand","attrs":{"title":"Verify data collection"},"content":[{"type":"paragraph","content":[{"text":"Once the collector has been launched, it is important to check if the ingestion is performed in a proper way. To do so, go to the collector’s logs console.","type":"text"}]},{"type":"paragraph","content":[{"text":"This service has the following components:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"a8e5adb9-3400-4488-b0a0-d58f3c54af81"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Component","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Setup","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of authenticating the service and managing the token expiration when needed.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Puller","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of pulling the data in a organized way and delivering the events via SDK.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Setup output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful run has the following output messages for the setup module:","type":"text"}]},{"type":"codeBlock","content":[{"text":"ProofPointPuller(proofpoint_tap,123456,clicksBlocked,predefined) - Starting thread\n2023-12-21T19:10:16.127 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksBlocked,predefined) -> Waiting until setup will be executed\n2023-12-21T19:10:16.127 INFO InputProcess::MainThread -> ServiceThread(proofpoint_tap,123456,clicksPermitted,predefined) - Starting thread (execution_period=60s)\n2023-12-21T19:10:16.127 INFO InputProcess::MainThread -> ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,clicksPermitted#predefined) -> Starting thread\n2023-12-21T19:10:16.127 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) - Starting thread\n2023-12-21T19:10:16.127 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksPermitted,predefined) -> Waiting until setup will be executed\n2023-12-21T19:10:16.127 WARNING InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,clicksPermitted#predefined) -> The token/header/authentication has not been created yet\n2023-12-21T19:10:16.127 INFO InputProcess::MainThread -> ServiceThread(proofpoint_tap,123456,messagesBlocked,predefined) - Starting thread (execution_period=60s)\n2023-12-21T19:10:16.128 INFO InputProcess::MainThread -> ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesBlocked#predefined) -> Starting thread\n2023-12-21T19:10:16.129 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) - Starting thread\n2023-12-21T19:10:16.129 WARNING InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesBlocked#predefined) -> The token/header/authentication has not been created yet\n2023-12-21T19:10:16.130 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) -> Waiting until setup will be executed\n2023-12-21T19:10:16.131 INFO InputProcess::MainThread -> ServiceThread(proofpoint_tap,123456,messagesDelivered,predefined) - Starting thread (execution_period=60s)\n2023-12-21T19:10:16.131 INFO InputProcess::MainThread -> ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> Starting thread\n2023-12-21T19:10:16.131 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) - Starting thread\n2023-12-21T19:10:16.131 WARNING InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> The token/header/authentication has not been created yet\n2023-12-21T19:10:16.131 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> Waiting until setup will be executed\n2023-12-21T19:10:16.145 INFO InputProcess::MainThread -> [GC] global: 32.2% -> 32.2%, process: RSS(40.54MiB -> 40.54MiB), VMS(1.05GiB -> 1.05GiB)\n2023-12-21T19:10:16.768 INFO OutputProcess::DevoSender(internal_senders,devo_sender_0) -> Created a sender: {\"name\": \"DevoSender(internal_senders,devo_sender_0)\", \"url\": \"collector-eu.devo.io:443\", \"chain_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/chain.crt\", \"cert_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.crt\", \"key_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.key\", \"transport_layer_type\": \"SSL\", \"last_usage_timestamp\": null, \"socket_status\": null}, hostname: \"2023-apac-0046\", session_id: \"140555598948768\"\n2023-12-21T19:10:18.447 INFO InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,clicksPermitted#predefined) -> Setup for module has been successfully executed","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Puller output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful initial run has the following output messages for the puller module:","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note that the ","type":"text"},{"text":"PrePull","type":"text","marks":[{"type":"code"}]},{"text":" action is executed only one time before the first run of the ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"2023-12-22T10:24:02.739 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksBlocked,predefined) -> Pull Started\n2023-12-22T10:24:02.741 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksBlocked,predefined) -> Time Window FROM: 2023-12-10 11:00:00+00:00 TO: 2023-12-10 12:00:00+00:00\n2023-12-22T10:24:02.742 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksBlocked,predefined) -> Start_Time_in_utc must be at most 7.00d into the past, Changing the time startTime to be in the specified time\n2023-12-22T10:24:04.567 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksBlocked,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703220842731):Number of requests made: 1; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-22T10:24:04.568 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksBlocked,predefined) -> Time Window FROM: 2023-12-15 05:54:02.731674+00:00 TO: 2023-12-15 06:54:02.731674+00:00\n2023-12-22T10:24:06.173 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksBlocked,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703220842731):Number of requests made: 2; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.\n2023-12-22T10:24:06.174 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksBlocked,predefined) -> Time Window FROM: 2023-12-15 06:54:02.731674+00:00 TO: 2023-12-15 07:54:02.731674+00:00\n2023-12-22T10:24:07.839 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksBlocked,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703220842731):Number of requests made: 3; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.","type":"text"}]},{"type":"paragraph","content":[{"text":"After a successful collector’s execution (that is, no error logs found), you will see the following log message:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-26T08:57:42.836 INFO InputProcess::ProofPointPuller(proofpoint_tap,123456,clicksBlocked,predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1703561242970):Number of requests made: 6; Number of events received: 0; Number of duplicated events filtered out: 0; Number of events generated and sent: 0; Average of events per second: 0.000.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"The value ","type":"text"},{"text":"@devo_pulling_id","type":"text","marks":[{"type":"code"}]},{"text":" is injected in each event to group all events ingested by the same pull action. You can use it to get the exact events downloaded in that ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action in Devo’s search window.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Restart the persistence"},"content":[{"type":"paragraph","content":[{"text":"This collector uses persistent storage to download events in an orderly fashion and avoid duplicates. In case you want to re-ingest historical data or recreate the persistence, you can restart the persistence of this collector by following these steps:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Edit the configuration file.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Change the value of the ","type":"text"},{"text":"start_time_in_utc_format","type":"text","marks":[{"type":"code"}]},{"text":" parameter to a different one.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Save the changes.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Restart the collector.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"The collector will detect this change and will restart the persistence using the parameters of the configuration file or the default configuration in case it has not been provided.","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Note that this action clears the persistence and cannot be recovered in any way. Resetting persistence could result in duplicate or lost events.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Troubleshooting"},"content":[{"type":"paragraph","content":[{"text":"This collector has different security layers that detect both an invalid configuration and abnormal operation. This table will help you detect and resolve the most common errors.","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"e217785e-578d-47c2-8a9f-35e0699c9600"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[144.0]},"content":[{"type":"paragraph","content":[{"text":"Error type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[95.0]},"content":[{"type":"paragraph","content":[{"text":"Error ID","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[210.0]},"content":[{"type":"paragraph","content":[{"text":"Error message","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[141.0]},"content":[{"type":"paragraph","content":[{"text":"Cause","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Solution","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[144.0]},"content":[{"type":"paragraph","content":[{"text":"SetupError","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[95.0]},"content":[{"type":"paragraph","content":[{"text":"100","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[210.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[141.0]},"content":[{"type":"paragraph","content":[{"text":"username and password is not correct","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure that credentials are correct.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[95.0]},"content":[{"type":"paragraph","content":[{"text":"101","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[210.0]},"content":[{"type":"paragraph","content":[{"text":"Error occurred while retrieving events from ProofPoint server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[141.0]},"content":[{"type":"paragraph","content":[{"text":"start_time_in_utc","type":"text","marks":[{"type":"code"}]},{"text":" is in future or not in proper format","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure the start time is not in future and not in proper format","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[144.0]},"content":[{"type":"paragraph","content":[{"text":"PullError","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[95.0]},"content":[{"type":"paragraph","content":[{"text":"300","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[210.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server : {summery} , {details}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[141.0]},"content":[{"type":"paragraph","content":[{"text":"This error happens when the collector tries to fetch the data from API.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"In this error you will find the HTTP error code as well as the summary and details.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[144.0]},"content":[{"type":"paragraph","content":[{"text":" ","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[95.0]},"content":[{"type":"paragraph","content":[{"text":"301","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[210.0]},"content":[{"type":"paragraph","content":[{"text":"Some Error occurred while retrieving events from ProofPoint server : {Exception}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[141.0]},"content":[{"type":"paragraph","content":[{"text":"Some exception occured while making the API request.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Reach out to the developer with the exact error message.","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Threats","type":"text"}]},{"type":"expand","attrs":{"title":"Verify data collection"},"content":[{"type":"paragraph","content":[{"text":"Once the collector has been launched, it is important to check if the ingestion is performed in a proper way. To do so, go to the collector’s logs console.","type":"text"}]},{"type":"paragraph","content":[{"text":"This service has the following components:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"5a2d40f9-d559-4c23-a959-1ca337be2f2e"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Component","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Setup","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of authenticating the service and managing the token expiration when needed.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Puller","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of pulling the data in a organized way and delivering the events via SDK.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Setup output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful run has the following output messages for the setup module:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-26T09:15:01.181 INFO OutputProcess::MainThread -> Process started\n2023-12-26T09:15:01.182 INFO MainProcess::MainThread -> Started all object from \"MainProcess\" process\n2023-12-26T09:15:01.182 INFO InputProcess::MainThread -> Process Started\n2023-12-26T09:15:01.204 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) Starting the execution of init_variables()\n2023-12-26T09:15:01.204 INFO InputProcess::MainThread -> Validating service metadata\n2023-12-26T09:15:01.206 INFO InputProcess::MainThread -> Validating defined module definition\n2023-12-26T09:15:01.207 INFO OutputProcess::MainThread -> DevoSender(standard_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(standard_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManager(standard_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSender(lookup_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(lookup_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManager(lookup_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputStandardConsumer(standard_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputStandardConsumer(standard_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO InputProcess::MainThread -> Validating common input config\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(standard_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(standard_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(lookup_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(lookup_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSender(internal_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputLookupConsumer(lookup_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputLookupConsumer(lookup_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(internal_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSenderManager(internal_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(internal_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(internal_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputInternalConsumer(internal_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputInternalConsumer(internal_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Validating service input config\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Running overriding rules\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Validating the rate limiter config given by the user\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> setting has not been defined. The generic settings will be used instead.\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Adding raw config to the collector store\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Running custom validation rules\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Creating API client.\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Created request client: \n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,threats,predefined) Finalizing the execution of init_variables()\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> InputThread(proofpoint_tap,123456) - Starting thread (execution_period=60s)\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ServiceThread(proofpoint_tap,123456,threats,predefined) - Starting thread (execution_period=60s)\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,threats#predefined) -> Starting thread\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) - Starting thread\n2023-12-26T09:15:01.212 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> Waiting until setup will be executed\n2023-12-26T09:15:01.212 WARNING InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,threats#predefined) -> The token/header/authentication has not been created yet\n2023-12-26T09:15:01.221 INFO OutputProcess::MainThread -> [GC] global: 28.0% -> 28.1%, process: RSS(40.77MiB -> 41.89MiB), VMS(926.00MiB -> 926.00MiB)\n2023-12-26T09:15:01.225 INFO InputProcess::MainThread -> [GC] global: 28.0% -> 28.1%, process: RSS(40.29MiB -> 40.29MiB), VMS(421.98MiB -> 421.98MiB)\n2023-12-26T09:15:01.713 INFO OutputProcess::DevoSender(internal_senders,devo_sender_0) -> Created a sender: {\"name\": \"DevoSender(internal_senders,devo_sender_0)\", \"url\": \"collector-eu.devo.io:443\", \"chain_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/chain.crt\", \"cert_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.crt\", \"key_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.key\", \"transport_layer_type\": \"SSL\", \"last_usage_timestamp\": null, \"socket_status\": null}, hostname: \"2023-apac-0046\", session_id: \"139871239825152\"\n2023-12-26T09:15:03.324 INFO InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,threats#predefined) -> Setup for module has been successfully executed","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Puller output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful initial run has the following output messages for the puller module:","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note that the ","type":"text"},{"text":"PrePull","type":"text","marks":[{"type":"code"}]},{"text":" action is executed only one time before the first run of the ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"2025-03-05T16:28:00.760 INFO InputProcess::ProofPointThreatsPuller(proofpoint_tap#1212090,threats#predefined) -> Pull Started\n2025-03-05T16:28:00.762 INFO InputProcess::ProofPointThreatsPuller(proofpoint_tap#1212090,threats#predefined) -> Time Window FROM: 2024-09-26 14:00:00+00:00 TO: 2024-09-26 15:00:00+00:00\n2025-03-05T16:59:32.041 INFO InputProcess::ProofPointThreatsPuller(proofpoint_tap#1212090,threats#predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1741170569801):Number of requests made: 1; Number of events received: 193; Number of duplicated events filtered out: 187; Number of events generated and sent: 6; Average of events per second: 2.687.\n2025-03-05T16:59:32.041 INFO InputProcess::ProofPointThreatsPuller(proofpoint_tap#1212090,threats#predefined) -> Statistics for this pull cycle (@devo_pulling_id=1741170569801):Number of requests made: 1; Number of events received: 193; Number of duplicated events filtered out: 187; Number of events generated and sent: 6; Average of events per second: 2.687.","type":"text"}]},{"type":"paragraph","content":[{"text":"After a successful collector’s execution (that is, no error logs found), you will see the following log message:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2025-03-05T16:59:32.041 INFO InputProcess::ProofPointThreatsPuller(proofpoint_tap#1212090,threats#predefined) -> Statistics for this pull cycle (@devo_pulling_id=1741170569801):Number of requests made: 1; Number of events received: 193; Number of duplicated events filtered out: 187; Number of events generated and sent: 6; Average of events per second: 2.687.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"The value ","type":"text"},{"text":"@devo_pulling_id","type":"text","marks":[{"type":"code"}]},{"text":" is injected in each event to group all events ingested by the same pull action. You can use it to get the exact events downloaded in that ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action in Devo’s search window.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Restart the persistence"},"content":[{"type":"paragraph","content":[{"text":"This collector uses persistent storage to download events in an orderly fashion and avoid duplicates. In case you want to re-ingest historical data or recreate the persistence, you can restart the persistence of this collector by following these steps:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Edit the configuration file.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Change the value of the ","type":"text"},{"text":"start_time_in_utc_format","type":"text","marks":[{"type":"code"}]},{"text":" parameter to a different one.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Save the changes.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Restart the collector.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"The collector will detect this change and will restart the persistence using the parameters of the configuration file or the default configuration in case it has not been provided.","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Note that this action clears the persistence and cannot be recovered in any way. Resetting persistence could result in duplicate or lost events.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Troubleshooting"},"content":[{"type":"paragraph","content":[{"text":"This collector has different security layers that detect both an invalid configuration and abnormal operation. This table will help you detect and resolve the most common errors.","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"bc7d776c-84ae-4086-941e-c9d11cdbcd94"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Error type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error ID","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error message","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Cause","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Solution","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"SetupError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"100","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"username and password is not correct","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure that credentials are correct.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"101","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error occurred while retrieving events from ProofPoint server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"start_time_in_utc is in future or not in proper format","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure the start time is not in future and not in proper format","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"PullError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"300","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server : {summery} , {details}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"This error happens when the collector tries to fetch the data from API.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"In this error you will find the HTTP error code as well as the summary and details.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":" ","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"301","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some Error occurred while retrieving events from ProofPoint server : {Exception}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some exception occured while making the API request.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Reach out to the developer with the exact error message.","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Campaigns","type":"text"}]},{"type":"expand","attrs":{"title":"Verify data collection"},"content":[{"type":"paragraph","content":[{"text":"Once the collector has been launched, it is important to check if the ingestion is performed in a proper way. To do so, go to the collector’s logs console.","type":"text"}]},{"type":"paragraph","content":[{"text":"This service has the following components:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"cd4e6be2-8f80-4f8b-aa9e-be7520325592"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Component","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Setup","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of authenticating the service and managing the token expiration when needed.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Puller","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of pulling the data in a organized way and delivering the events via SDK.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Setup output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful run has the following output messages for the setup module:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-26T09:15:01.181 INFO OutputProcess::MainThread -> Process started\n2023-12-26T09:15:01.182 INFO MainProcess::MainThread -> Started all object from \"MainProcess\" process\n2023-12-26T09:15:01.182 INFO InputProcess::MainThread -> Process Started\n2023-12-26T09:15:01.204 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) Starting the execution of init_variables()\n2023-12-26T09:15:01.204 INFO InputProcess::MainThread -> Validating service metadata\n2023-12-26T09:15:01.206 INFO InputProcess::MainThread -> Validating defined module definition\n2023-12-26T09:15:01.207 INFO OutputProcess::MainThread -> DevoSender(standard_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(standard_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManager(standard_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSender(lookup_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(lookup_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManager(lookup_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputStandardConsumer(standard_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputStandardConsumer(standard_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO InputProcess::MainThread -> Validating common input config\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(standard_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(standard_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(lookup_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(lookup_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSender(internal_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputLookupConsumer(lookup_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputLookupConsumer(lookup_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(internal_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSenderManager(internal_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(internal_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(internal_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputInternalConsumer(internal_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputInternalConsumer(internal_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Validating service input config\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Running overriding rules\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Validating the rate limiter config given by the user\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> setting has not been defined. The generic settings will be used instead.\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Adding raw config to the collector store\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Running custom validation rules\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Creating API client.\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Created request client: \n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) Finalizing the execution of init_variables()\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> InputThread(proofpoint_tap,123456) - Starting thread (execution_period=60s)\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ServiceThread(proofpoint_tap,123456,messagesDelivered,predefined) - Starting thread (execution_period=60s)\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> Starting thread\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) - Starting thread\n2023-12-26T09:15:01.212 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> Waiting until setup will be executed\n2023-12-26T09:15:01.212 WARNING InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> The token/header/authentication has not been created yet\n2023-12-26T09:15:01.221 INFO OutputProcess::MainThread -> [GC] global: 28.0% -> 28.1%, process: RSS(40.77MiB -> 41.89MiB), VMS(926.00MiB -> 926.00MiB)\n2023-12-26T09:15:01.225 INFO InputProcess::MainThread -> [GC] global: 28.0% -> 28.1%, process: RSS(40.29MiB -> 40.29MiB), VMS(421.98MiB -> 421.98MiB)\n2023-12-26T09:15:01.713 INFO OutputProcess::DevoSender(internal_senders,devo_sender_0) -> Created a sender: {\"name\": \"DevoSender(internal_senders,devo_sender_0)\", \"url\": \"collector-eu.devo.io:443\", \"chain_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/chain.crt\", \"cert_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.crt\", \"key_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.key\", \"transport_layer_type\": \"SSL\", \"last_usage_timestamp\": null, \"socket_status\": null}, hostname: \"2023-apac-0046\", session_id: \"139871239825152\"\n2023-12-26T09:15:03.324 INFO InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> Setup for module has been successfully executed","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Puller output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful initial run has the following output messages for the puller module:","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note that the ","type":"text"},{"text":"PrePull","type":"text","marks":[{"type":"code"}]},{"text":" action is executed only one time before the first run of the ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"2025-03-05T16:28:01.769 INFO InputProcess::ProofPointCampaignsPuller(proofpoint_tap#1212090,campaigns#predefined) -> Pull Started\n2025-03-05T16:28:01.770 INFO InputProcess::ProofPointCampaignsPuller(proofpoint_tap#1212090,campaigns#predefined) -> Time Window FROM: 2025-02-25 11:30:29+00:00 TO: 2025-02-26 11:29:29+00:00\n2025-03-05T16:28:07.447 INFO InputProcess::ProofPointCampaignsPuller(proofpoint_tap#1212090,campaigns#predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1741172281758):Number of requests made: 1; Number of events received: 359; Number of duplicated events filtered out: 359; Number of events generated and sent: 0; Average of events per second: 0.000.\n2025-03-05T16:28:07.447 INFO InputProcess::ProofPointCampaignsPuller(proofpoint_tap#1212090,campaigns#predefined) -> Statistics for this pull cycle (@devo_pulling_id=1741172281758):Number of requests made: 1; Number of events received: 359; Number of duplicated events filtered out: 359; Number of events generated and sent: 0; Average of events per second: 0.000.","type":"text"}]},{"type":"paragraph","content":[{"text":"After a successful collector’s execution (that is, no error logs found), you will see the following log message:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2025-03-05T16:28:07.447 INFO InputProcess::ProofPointCampaignsPuller(proofpoint_tap#1212090,campaigns#predefined) -> Statistics for this pull cycle (@devo_pulling_id=1741172281758):Number of requests made: 1; Number of events received: 359; Number of duplicated events filtered out: 359; Number of events generated and sent: 0; Average of events per second: 0.000.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"The value ","type":"text"},{"text":"@devo_pulling_id","type":"text","marks":[{"type":"code"}]},{"text":" is injected in each event to group all events ingested by the same pull action. You can use it to get the exact events downloaded in that ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action in Devo’s search window.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Restart the persistence"},"content":[{"type":"paragraph","content":[{"text":"This collector uses persistent storage to download events in an orderly fashion and avoid duplicates. In case you want to re-ingest historical data or recreate the persistence, you can restart the persistence of this collector by following these steps:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Edit the configuration file.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Change the value of the ","type":"text"},{"text":"start_time_in_utc_format","type":"text","marks":[{"type":"code"}]},{"text":" parameter to a different one.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Save the changes.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Restart the collector.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"The collector will detect this change and will restart the persistence using the parameters of the configuration file or the default configuration in case it has not been provided.","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Note that this action clears the persistence and cannot be recovered in any way. Resetting persistence could result in duplicate or lost events.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Troubleshooting"},"content":[{"type":"paragraph","content":[{"text":"This collector has different security layers that detect both an invalid configuration and abnormal operation. This table will help you detect and resolve the most common errors.","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"b40923b5-32b6-4164-840d-c9dd3a26d626"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Error type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error ID","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error message","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Cause","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Solution","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"SetupError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"100","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"username and password is not correct","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure that credentials are correct.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"101","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error occurred while retrieving events from ProofPoint server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"start_time_in_utc is in future or not in proper format","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure the start time is not in future and not in proper format","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"PullError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"300","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server : {summery} , {details}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"This error happens when the collector tries to fetch the data from API.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"In this error you will find the HTTP error code as well as the summary and details.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":" ","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"301","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some Error occurred while retrieving events from ProofPoint server : {Exception}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some exception occured while making the API request.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Reach out to the developer with the exact error message.","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"People Topclicks","type":"text"}]},{"type":"expand","attrs":{"title":"Verify data collection"},"content":[{"type":"paragraph","content":[{"text":"Once the collector has been launched, it is important to check if the ingestion is performed in a proper way. To do so, go to the collector’s logs console.","type":"text"}]},{"type":"paragraph","content":[{"text":"This service has the following components:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"d74b20e4-0471-4193-b599-6820a761b2d8"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Component","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Setup","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of authenticating the service and managing the token expiration when needed.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Puller","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of pulling the data in a organized way and delivering the events via SDK.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Setup output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful run has the following output messages for the setup module:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-26T09:15:01.181 INFO OutputProcess::MainThread -> Process started\n2023-12-26T09:15:01.182 INFO MainProcess::MainThread -> Started all object from \"MainProcess\" process\n2023-12-26T09:15:01.182 INFO InputProcess::MainThread -> Process Started\n2023-12-26T09:15:01.204 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) Starting the execution of init_variables()\n2023-12-26T09:15:01.204 INFO InputProcess::MainThread -> Validating service metadata\n2023-12-26T09:15:01.206 INFO InputProcess::MainThread -> Validating defined module definition\n2023-12-26T09:15:01.207 INFO OutputProcess::MainThread -> DevoSender(standard_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(standard_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManager(standard_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSender(lookup_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(lookup_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManager(lookup_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputStandardConsumer(standard_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputStandardConsumer(standard_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO InputProcess::MainThread -> Validating common input config\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(standard_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(standard_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(lookup_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(lookup_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSender(internal_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputLookupConsumer(lookup_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputLookupConsumer(lookup_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(internal_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSenderManager(internal_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(internal_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(internal_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputInternalConsumer(internal_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputInternalConsumer(internal_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Validating service input config\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Running overriding rules\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Validating the rate limiter config given by the user\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> setting has not been defined. The generic settings will be used instead.\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Adding raw config to the collector store\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Running custom validation rules\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Creating API client.\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Created request client: \n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) Finalizing the execution of init_variables()\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> InputThread(proofpoint_tap,123456) - Starting thread (execution_period=60s)\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ServiceThread(proofpoint_tap,123456,messagesDelivered,predefined) - Starting thread (execution_period=60s)\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> Starting thread\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) - Starting thread\n2023-12-26T09:15:01.212 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> Waiting until setup will be executed\n2023-12-26T09:15:01.212 WARNING InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> The token/header/authentication has not been created yet\n2023-12-26T09:15:01.221 INFO OutputProcess::MainThread -> [GC] global: 28.0% -> 28.1%, process: RSS(40.77MiB -> 41.89MiB), VMS(926.00MiB -> 926.00MiB)\n2023-12-26T09:15:01.225 INFO InputProcess::MainThread -> [GC] global: 28.0% -> 28.1%, process: RSS(40.29MiB -> 40.29MiB), VMS(421.98MiB -> 421.98MiB)\n2023-12-26T09:15:01.713 INFO OutputProcess::DevoSender(internal_senders,devo_sender_0) -> Created a sender: {\"name\": \"DevoSender(internal_senders,devo_sender_0)\", \"url\": \"collector-eu.devo.io:443\", \"chain_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/chain.crt\", \"cert_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.crt\", \"key_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.key\", \"transport_layer_type\": \"SSL\", \"last_usage_timestamp\": null, \"socket_status\": null}, hostname: \"2023-apac-0046\", session_id: \"139871239825152\"\n2023-12-26T09:15:03.324 INFO InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> Setup for module has been successfully executed","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Puller output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful initial run has the following output messages for the puller module:","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note that the ","type":"text"},{"text":"PrePull","type":"text","marks":[{"type":"code"}]},{"text":" action is executed only one time before the first run of the ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"2025-03-05T15:59:40.797 INFO InputProcess::ProofPointPeoplePuller(proofpoint_tap#1212090,people_topclicks#predefined) -> Pull Started\n2025-03-05T15:59:40.799 INFO InputProcess::ProofPointPeoplePuller(proofpoint_tap#1212090,people_topclicks#predefined) -> Time Window FROM: 2025-02-25 21:48:55+00:00 TO: 2025-02-25 22:48:55+00:00\n2025-03-05T15:59:44.243 INFO InputProcess::ProofPointPeoplePuller(proofpoint_tap#1212090,people_topclicks#predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1741170580790):Number of requests made: 1; Number of events received: 31; Number of duplicated events filtered out: 31; Number of events generated and sent: 0; Average of events per second: 0.000.\n2025-03-05T15:59:44.243 INFO InputProcess::ProofPointPeoplePuller(proofpoint_tap#1212090,people_topclicks#predefined) -> Statistics for this pull cycle (@devo_pulling_id=1741170580790):Number of requests made: 1; Number of events received: 31; Number of duplicated events filtered out: 31; Number of events generated and sent: 0; Average of events per second: 0.000.","type":"text"}]},{"type":"paragraph","content":[{"text":"After a successful collector’s execution (that is, no error logs found), you will see the following log message:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2025-03-05T15:59:44.243 INFO InputProcess::ProofPointPeoplePuller(proofpoint_tap#1212090,people_topclicks#predefined) -> Statistics for this pull cycle (@devo_pulling_id=1741170580790):Number of requests made: 1; Number of events received: 31; Number of duplicated events filtered out: 31; Number of events generated and sent: 0; Average of events per second: 0.000.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"The value ","type":"text"},{"text":"@devo_pulling_id","type":"text","marks":[{"type":"code"}]},{"text":" is injected in each event to group all events ingested by the same pull action. You can use it to get the exact events downloaded in that ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action in Devo’s search window.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Restart the persistence"},"content":[{"type":"paragraph","content":[{"text":"This collector uses persistent storage to download events in an orderly fashion and avoid duplicates. In case you want to re-ingest historical data or recreate the persistence, you can restart the persistence of this collector by following these steps:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Edit the configuration file.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Change the value of the ","type":"text"},{"text":"start_time_in_utc_format","type":"text","marks":[{"type":"code"}]},{"text":" parameter to a different one.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Save the changes.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Restart the collector.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"The collector will detect this change and will restart the persistence using the parameters of the configuration file or the default configuration in case it has not been provided.","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Note that this action clears the persistence and cannot be recovered in any way. Resetting persistence could result in duplicate or lost events.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Troubleshooting"},"content":[{"type":"paragraph","content":[{"text":"This collector has different security layers that detect both an invalid configuration and abnormal operation. This table will help you detect and resolve the most common errors.","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"3ee2917f-ddb8-4afb-abc6-db4e0351c0b1"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Error type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error ID","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error message","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Cause","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Solution","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"SetupError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"100","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"username and password is not correct","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure that credentials are correct.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"101","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error occurred while retrieving events from ProofPoint server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"start_time_in_utc is in future or not in proper format","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure the start time is not in future and not in proper format","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"PullError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"300","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server : {summery} , {details}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"This error happens when the collector tries to fetch the data from API.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"In this error you will find the HTTP error code as well as the summary and details.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":" ","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"301","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some Error occurred while retrieving events from ProofPoint server : {Exception}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some exception occured while making the API request.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Reach out to the developer with the exact error message.","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"People VAP","type":"text","marks":[{"type":"strong"}]}]},{"type":"expand","attrs":{"title":"Verify data collection"},"content":[{"type":"paragraph","content":[{"text":"Once the collector has been launched, it is important to check if the ingestion is performed in a proper way. To do so, go to the collector’s logs console.","type":"text"}]},{"type":"paragraph","content":[{"text":"This service has the following components:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"31e59143-6e7d-4c60-9675-fa2d05c44ec8"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Component","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Setup","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of authenticating the service and managing the token expiration when needed.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[231.0]},"content":[{"type":"paragraph","content":[{"text":"Puller","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[499.0]},"content":[{"type":"paragraph","content":[{"text":"The setup module is in charge of pulling the data in a organized way and delivering the events via SDK.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Setup output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful run has the following output messages for the setup module:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-26T09:15:01.181 INFO OutputProcess::MainThread -> Process started\n2023-12-26T09:15:01.182 INFO MainProcess::MainThread -> Started all object from \"MainProcess\" process\n2023-12-26T09:15:01.182 INFO InputProcess::MainThread -> Process Started\n2023-12-26T09:15:01.204 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) Starting the execution of init_variables()\n2023-12-26T09:15:01.204 INFO InputProcess::MainThread -> Validating service metadata\n2023-12-26T09:15:01.206 INFO InputProcess::MainThread -> Validating defined module definition\n2023-12-26T09:15:01.207 INFO OutputProcess::MainThread -> DevoSender(standard_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(standard_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManager(standard_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSender(lookup_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(lookup_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.208 INFO OutputProcess::MainThread -> DevoSenderManager(lookup_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputStandardConsumer(standard_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputStandardConsumer(standard_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO InputProcess::MainThread -> Validating common input config\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(standard_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(standard_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(lookup_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(lookup_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSender(internal_senders,devo_sender_0) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputLookupConsumer(lookup_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputLookupConsumer(lookup_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSenderManagerMonitor(internal_senders,devo_us_1) -> Starting thread (every 300 seconds)\n2023-12-26T09:15:01.209 INFO OutputProcess::MainThread -> DevoSenderManager(internal_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:15:01.209 INFO OutputProcess::DevoSenderManager(internal_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(internal_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.209 INFO OutputProcess::OutputInternalConsumer(internal_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputInternalConsumer(internal_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Validating service input config\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Running overriding rules\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> Validating the rate limiter config given by the user\n2023-12-26T09:15:01.210 INFO InputProcess::MainThread -> setting has not been defined. The generic settings will be used instead.\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Adding raw config to the collector store\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Running custom validation rules\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Creating API client.\n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> Created request client: \n2023-12-26T09:15:01.211 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) Finalizing the execution of init_variables()\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> InputThread(proofpoint_tap,123456) - Starting thread (execution_period=60s)\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ServiceThread(proofpoint_tap,123456,messagesDelivered,predefined) - Starting thread (execution_period=60s)\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> Starting thread\n2023-12-26T09:15:01.212 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) - Starting thread\n2023-12-26T09:15:01.212 WARNING InputProcess::ProofPointPuller(proofpoint_tap,123456,messagesDelivered,predefined) -> Waiting until setup will be executed\n2023-12-26T09:15:01.212 WARNING InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> The token/header/authentication has not been created yet\n2023-12-26T09:15:01.221 INFO OutputProcess::MainThread -> [GC] global: 28.0% -> 28.1%, process: RSS(40.77MiB -> 41.89MiB), VMS(926.00MiB -> 926.00MiB)\n2023-12-26T09:15:01.225 INFO InputProcess::MainThread -> [GC] global: 28.0% -> 28.1%, process: RSS(40.29MiB -> 40.29MiB), VMS(421.98MiB -> 421.98MiB)\n2023-12-26T09:15:01.713 INFO OutputProcess::DevoSender(internal_senders,devo_sender_0) -> Created a sender: {\"name\": \"DevoSender(internal_senders,devo_sender_0)\", \"url\": \"collector-eu.devo.io:443\", \"chain_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/chain.crt\", \"cert_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.crt\", \"key_path\": \"/home/mdtausif/Gitlab/devo-collector-proofpoint-tap/certs/int-if-integrations-india.key\", \"transport_layer_type\": \"SSL\", \"last_usage_timestamp\": null, \"socket_status\": null}, hostname: \"2023-apac-0046\", session_id: \"139871239825152\"\n2023-12-26T09:15:03.324 INFO InputProcess::ProofPointPullerSetup(proofpoint_tap_collector,proofpoint_tap#123456,messagesDelivered#predefined) -> Setup for module has been successfully executed","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Puller output","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful initial run has the following output messages for the puller module:","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note that the ","type":"text"},{"text":"PrePull","type":"text","marks":[{"type":"code"}]},{"text":" action is executed only one time before the first run of the ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"2025-03-05T15:59:29.807 INFO InputProcess::ProofPointPeoplePuller(proofpoint_tap#1212090,people_vap#predefined) -> Pull Started\n2025-03-05T15:59:29.808 INFO InputProcess::ProofPointPeoplePuller(proofpoint_tap#1212090,people_vap#predefined) -> Time Window FROM: 2025-02-25 21:48:56+00:00 TO: 2025-02-25 22:48:56+00:00023-12-26T09:15:12.140 \n2025-03-05T15:59:32.041 INFO InputProcess::ProofPointPeoplePuller(proofpoint_tap#1212090,people_vap#predefined) -> (Partial) Statistics for this pull cycle (@devo_pulling_id=1741170569801):Number of requests made: 1; Number of events received: 193; Number of duplicated events filtered out: 187; Number of events generated and sent: 6; Average of events per second: 2.687.\n2025-03-05T15:59:32.041 INFO InputProcess::ProofPointPeoplePuller(proofpoint_tap#1212090,people_vap#predefined) -> Statistics for this pull cycle (@devo_pulling_id=1741170569801):Number of requests made: 1; Number of events received: 193; Number of duplicated events filtered out: 187; Number of events generated and sent: 6; Average of events per second: 2.687.","type":"text"}]},{"type":"paragraph","content":[{"text":"After a successful collector’s execution (that is, no error logs found), you will see the following log message:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2025-03-05T15:59:32.041 INFO InputProcess::ProofPointPeoplePuller(proofpoint_tap#1212090,people_vap#predefined) -> Statistics for this pull cycle (@devo_pulling_id=1741170569801):Number of requests made: 1; Number of events received: 193; Number of duplicated events filtered out: 187; Number of events generated and sent: 6; Average of events per second: 2.687.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"The value ","type":"text"},{"text":"@devo_pulling_id","type":"text","marks":[{"type":"code"}]},{"text":" is injected in each event to group all events ingested by the same pull action. You can use it to get the exact events downloaded in that ","type":"text"},{"text":"Pull","type":"text","marks":[{"type":"code"}]},{"text":" action in Devo’s search window.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Restart the persistence"},"content":[{"type":"paragraph","content":[{"text":"This collector uses persistent storage to download events in an orderly fashion and avoid duplicates. In case you want to re-ingest historical data or recreate the persistence, you can restart the persistence of this collector by following these steps:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Edit the configuration file.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Change the value of the ","type":"text"},{"text":"start_time_in_utc_format","type":"text","marks":[{"type":"code"}]},{"text":" parameter to a different one.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Save the changes.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Restart the collector.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"The collector will detect this change and will restart the persistence using the parameters of the configuration file or the default configuration in case it has not been provided.","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Note that this action clears the persistence and cannot be recovered in any way. Resetting persistence could result in duplicate or lost events.","type":"text"}]}]}]},{"type":"expand","attrs":{"title":"Troubleshooting"},"content":[{"type":"paragraph","content":[{"text":"This collector has different security layers that detect both an invalid configuration and abnormal operation. This table will help you detect and resolve the most common errors.","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"6a327e87-ed7e-426b-9b6e-7b35e8ff1c33"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Error type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error ID","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error message","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Cause","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-accent-gray-subtlest, #F4F5F7)","rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Solution","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"SetupError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"100","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"username and password is not correct","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure that credentials are correct.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"101","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Error occurred while retrieving events from ProofPoint server","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"start_time_in_utc is in future or not in proper format","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Make sure the start time is not in future and not in proper format","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"PullError","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"300","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"HTTP Error occurred while retrieving events from Proof point server : {summery} , {details}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"This error happens when the collector tries to fetch the data from API.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"In this error you will find the HTTP error code as well as the summary and details.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":" ","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"301","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some Error occurred while retrieving events from ProofPoint server : {Exception}","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Some exception occured while making the API request.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[140.0]},"content":[{"type":"paragraph","content":[{"text":"Reach out to the developer with the exact error message.","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Collector operations","type":"text"}]},{"type":"paragraph","content":[{"text":"This section is intended to explain how to proceed with specific operations of this collector.","type":"text"}]},{"type":"expand","attrs":{"title":"Verify collector operations"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Initialization","type":"text"}]},{"type":"paragraph","content":[{"text":"The initialization module is in charge of setup and running the input (pulling logic) and output (delivering logic) services and validating the given configuration.","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful run has the following output messages for the initializer module:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-26T09:05:33.935 INFO OutputProcess::MainThread -> DevoSenderManager(internal_senders,manager,devo_us_1) -> Starting thread\n2023-12-26T09:05:33.935 INFO InputProcess::MainThread -> Validating defined module definition\n2023-12-26T09:05:33.936 INFO OutputProcess::DevoSenderManager(internal_senders,manager,devo_us_1) -> [EMERGENCY PERSISTENCE SYSTEM] DevoSenderManager(internal_senders,manager,devo_us_1) -> Nothing retrieved from the persistence.\n2023-12-26T09:05:33.936 INFO OutputProcess::OutputInternalConsumer(internal_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputInternalConsumer(internal_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:05:33.936 INFO OutputProcess::OutputLookupConsumer(lookup_senders_consumer_0) -> [EMERGENCY PERSISTENCE SYSTEM] OutputLookupConsumer(lookup_senders_consumer_0) -> Nothing retrieved from the persistence.\n2023-12-26T09:05:33.939 INFO InputProcess::MainThread -> Validating common input config\n2023-12-26T09:05:33.940 INFO InputProcess::MainThread -> Validating service input config\n2023-12-26T09:05:33.940 INFO InputProcess::MainThread -> Running overriding rules\n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> Validating the rate limiter config given by the user\n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> setting has not been defined. The generic settings will be used instead.\n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> Adding raw config to the collector store\n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> Running custom validation rules\n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> Creating API client.\n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> Created request client: \n2023-12-26T09:05:33.941 INFO InputProcess::MainThread -> ProofPointPuller(proofpoint_tap,123456,messagesBlocked,predefined) Finalizing the execution of init_variables()","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Events delivery and Devo ingestion","type":"text"}]},{"type":"paragraph","content":[{"text":"The event delivery module is in charge of receiving the events from the internal queues where all events are injected by the pullers and delivering them using the selected compatible delivery method.","type":"text"}]},{"type":"paragraph","content":[{"text":"A successful run has the following output messages for the initializer module:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2023-12-26T09:20:01.210 INFO OutputProcess::DevoSenderManagerMonitor(internal_senders,devo_us_1) -> Number of available senders: 1, sender manager internal queue size: 0\n2023-12-26T09:20:01.210 INFO OutputProcess::DevoSenderManagerMonitor(internal_senders,devo_us_1) -> enqueued_elapsed_times_in_seconds_stats: {}\n2023-12-26T09:20:01.210 INFO OutputProcess::DevoSenderManagerMonitor(internal_senders,devo_us_1) -> Sender: DevoSender(internal_senders,devo_sender_0), status: {\"internal_queue_size\": 0, \"is_connection_open\": True}\n2023-12-26T09:20:01.209 INFO OutputProcess::DevoSenderManagerMonitor(standard_senders,devo_us_1) -> Standard - Total number of messages: 0 messages/bytes sent since \"2023-12-26T03:45:01.203502+00:00\": 0/0, (elapsed 0.000 seconds)\n2023-12-26T09:20:01.209 INFO OutputProcess::DevoSenderManagerMonitor(lookup_senders,devo_us_1) -> Number of available senders: 1, sender manager internal queue size: 0\n2023-12-26T09:20:01.209 INFO OutputProcess::DevoSenderManagerMonitor(lookup_senders,devo_us_1) -> enqueued_elapsed_times_in_seconds_stats: {}\n2023-12-26T09:20:01.210 INFO OutputProcess::DevoSenderManagerMonitor(lookup_senders,devo_us_1) -> Sender: DevoSender(lookup_senders,devo_sender_0), status: {\"internal_queue_size\": 0, \"is_connection_open\": False}\n2023-12-26T09:20:01.210 INFO OutputProcess::DevoSenderManagerMonitor(lookup_senders,devo_us_1) -> Lookup - Total number of messages sent: 0, messages sent since \"2023-12-26 03:45:01.205552+00:00\": 0 (elapsed 0.000 seconds)","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"By default, these information traces will be displayed every 10 minutes.","type":"text"}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Sender services","type":"text"}]},{"type":"paragraph","content":[{"text":"The Integrations Factory Collector SDK has 3 different senders services depending on the event type to delivery (","type":"text"},{"text":"internal","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"standard","type":"text","marks":[{"type":"code"}]},{"text":", and ","type":"text"},{"text":"lookup","type":"text","marks":[{"type":"code"}]},{"text":"). This collector uses the following Sender Services:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"6bd704ef-0e1e-4407-992c-002264d15132"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-neutral, #F4F5F7)","rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"Sender services","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-neutral, #F4F5F7)","rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"internal_senders","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"In charge of delivering internal metrics to Devo such as logging traces or metrics.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"standard_senders","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"In charge of delivering pulled events to Devo.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Sender statistics","type":"text"}]},{"type":"paragraph","content":[{"text":"Each service displays its own performance statistics that allow checking how many events have been delivered to Devo by type:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"b4ce6589-85aa-4583-b31b-545216de40ff"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-neutral, #F4F5F7)","rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"Logging trace","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-neutral, #F4F5F7)","rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"Number of available senders: 1","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"Displays the number of concurrent senders available for the given Sender Service.","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"sender manager internal queue size: 0","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"Displays the items available in the internal sender queue.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"This value helps detect bottlenecks and needs to increase the performance of data delivery to Devo. This last can be made by increasing the concurrent senders.","type":"text"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"Total number of messages sent: 44, messages sent since \"2023-11-10 10:39:22.511671+00:00\": 21 (elapsed 0.007 seconds)","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[365.0]},"content":[{"type":"paragraph","content":[{"text":"Displayes the number of events from the last time and following the given example, the following conclusions can be obtained:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"44 events were sent to Devo since the collector started.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"The last checkpoint timestamp was ","type":"text"},{"text":"2022-06-28 10:39:22.511671+00:00","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"21 events where sent to Devo between the last UTC checkpoint and now.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Those 21 events required ","type":"text"},{"text":"0.007 seconds","type":"text","marks":[{"type":"code"}]},{"text":" to be delivered.","type":"text"}]}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"By default these traces will be shown every 10 minutes.","type":"text"}]}]}]}]}]}]},{"type":"expand","attrs":{"title":"Check memory usage"},"content":[{"type":"paragraph","content":[{"text":"To check the memory usage of this collector, look for the following log records in the collector which are displayed every 5 minutes by default, always after running the memory-free process.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"The used memory is displayed by running processes and the sum of both values will give the total used memory for the collector.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"The global pressure of the available memory is displayed in the ","type":"text"},{"text":"global","type":"text","marks":[{"type":"code"}]},{"text":" value.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"All metrics (Global, RSS, VMS) include the value before freeing and after ","type":"text"},{"text":"previous -> after freeing memory","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"codeBlock","content":[{"text":"2023-12-26T08:57:18.978 INFO InputProcess::MainThread -> [GC] global: 28.6% -> 28.7%, process: RSS(40.04MiB -> 40.04MiB), VMS(421.98MiB -> 421.98MiB)\n2023-12-26T08:57:18.979 INFO OutputProcess::MainThread -> [GC] global: 28.6% -> 28.7%, process: RSS(40.52MiB -> 41.39MiB), VMS(926.00MiB -> 926.00MiB)","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Differences between ","type":"text"},{"text":"RSS","type":"text","marks":[{"type":"code"}]},{"text":" and ","type":"text"},{"text":"VMS","type":"text","marks":[{"type":"code"}]},{"text":" memory usage:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"RSS","type":"text","marks":[{"type":"code"}]},{"text":" is the Resident Set Size, which is the actual physical memory the process is using","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"VMS","type":"text","marks":[{"type":"code"}]},{"text":" is the Virtual Memory Size which is the virtual memory that process is using","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Change log","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"ffe42a46-4f35-4aec-b7a9-e5bb47d1fad6"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-neutral, #F4F5F7)","rowspan":1,"colwidth":[129.0]},"content":[{"type":"paragraph","content":[{"text":"Release","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-neutral, #F4F5F7)","rowspan":1,"colwidth":[185.0]},"content":[{"type":"paragraph","content":[{"text":"Released on","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-neutral, #F4F5F7)","rowspan":1,"colwidth":[224.0]},"content":[{"type":"paragraph","content":[{"text":"Release type","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"background":"var(--ds-background-neutral, #F4F5F7)","rowspan":1,"colwidth":[222.0]},"content":[{"type":"paragraph","content":[{"text":"Recommendations","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[129.0]},"content":[{"type":"paragraph","content":[{"text":"v3.2.0","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[185.0]},"content":[{"type":"paragraph","content":[{"type":"date","attrs":{"timestamp":"1742515200000"}}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[224.0]},"content":[{"type":"paragraph","content":[{"type":"status","attrs":{"color":"yellow","style":"bold","text":"bug fixing","localId":"6a8c03f8-6e4a-4ce6-a380-68fb3f06d580"}},{"text":" ","type":"text"},{"type":"hardBreak"},{"type":"status","attrs":{"color":"blue","style":"bold","text":"IMPROVEMENTS","localId":"9dd59d2b-306c-4bf3-9386-80caef165557"}}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[222.0]},"content":[{"type":"paragraph","content":[{"text":"Recommended version","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":3,"rowspan":1,"colwidth":[185.0,224.0,222.0]},"content":[{"type":"paragraph"},{"type":"nestedExpand","attrs":{"title":"Details"},"content":[{"type":"paragraph","content":[{"text":"Improvements","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Refactor code and upgraded DCSDK to 1.15.0","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Differentiated error codes for SdkPersistenceServiceError.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Use of DOCKER_IMAGE environment variable to show docker_image property.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Now the property service_thread_execution_periods_in_seconds in collector_definition.yaml is optional.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":" Reduced Redis connections per collector to 2.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Implemented a mechanism to control if the certificates have expired.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Fixed CVE-2024-12797 vulnerability in cryptography library (updated from version 44.0.0 to 44.0.1).","type":"text"}]}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":" Upgraded docker base image to 1.4.1","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Bugs","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Fixed the OOMK bug causing the collector to restart","type":"text"}]}]}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[129.0]},"content":[{"type":"paragraph","content":[{"text":"v3.1.1","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[185.0]},"content":[{"type":"paragraph","content":[{"type":"date","attrs":{"timestamp":"1730160000000"}}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[224.0]},"content":[{"type":"paragraph","content":[{"type":"status","attrs":{"color":"yellow","style":"bold","text":"bug fixing","localId":"6d692c68-7fad-4d94-814d-feb51365f1fb"}}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[222.0]},"content":[{"type":"paragraph","content":[{"text":"Updates","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":3,"rowspan":1,"colwidth":[185.0,224.0,222.0]},"content":[{"type":"nestedExpand","attrs":{"title":"Details"},"content":[{"type":"paragraph","content":[{"text":"Bugs","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Fixed a bug related to request_time_in_seconds error for new endpoints","type":"text"}]}]}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[129.0]},"content":[{"type":"paragraph","content":[{"text":"v3.1.0","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[185.0]},"content":[{"type":"paragraph","content":[{"type":"date","attrs":{"timestamp":"1729728000000"}}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[224.0]},"content":[{"type":"paragraph","content":[{"type":"status","attrs":{"color":"blue","style":"bold","text":"IMPROVEMENTS","localId":"445025e7-112a-4b11-b6f8-5fc4c8415187"}},{"type":"hardBreak"},{"type":"status","attrs":{"color":"yellow","style":"bold","text":"bug fixing","localId":"38158e58-dc3d-40a2-b4bf-208d80b4577e"}},{"text":" ","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[222.0]},"content":[{"type":"paragraph","content":[{"text":"Updates","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":3,"rowspan":1,"colwidth":[185.0,224.0,222.0]},"content":[{"type":"nestedExpand","attrs":{"title":"Details"},"content":[{"type":"paragraph","content":[{"text":"Improvements","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Upgraded the DCSDK to v1.13.1","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Change internal queue management for protecting against OOMK","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Extracted ModuleThread structure from PullerAbstract","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Improve Controlled stop when both processes fails to instantiate","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Improve Controlled stop when InputProcess is killed","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Fixed error related a ValueError exception not well controlled","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Fixed error related with loss of some values in internal messages","type":"text"}]}]}]}]}]},{"type":"paragraph","content":[{"text":"Bugs","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Resolved the bug related to rate limit and repeated logs","type":"text"}]}]}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[129.0]},"content":[{"type":"paragraph","content":[{"text":"v3.0.0","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[185.0]},"content":[{"type":"paragraph","content":[{"type":"date","attrs":{"timestamp":"1724630400000"}}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[224.0]},"content":[{"type":"paragraph","content":[{"type":"status","attrs":{"color":"blue","style":"bold","text":"IMPROVEMENTS","localId":"1eca8019-cfb0-4248-9e02-4873a1072567"}},{"type":"hardBreak"},{"type":"status","attrs":{"color":"purple","style":"bold","text":"New Features","localId":"077ce48e-fc40-408a-acd3-f797aa896ab8"}},{"text":" ","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[222.0]},"content":[{"type":"paragraph","content":[{"text":"Update","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":3,"rowspan":1,"colwidth":[185.0,224.0,222.0]},"content":[{"type":"nestedExpand","attrs":{"title":"Details"},"content":[{"type":"paragraph","content":[{"text":"Improvements","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Upgraded the DCSDK from 1.11.0 to 1.12.4:","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Updated the base Docker Image to 1.3.0","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"New features","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Added 4 new enpdoints","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Threats","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Campaigns","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"People","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"vap","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"top-clickers","type":"text"}]}]}]}]}]}]}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[129.0]},"content":[{"type":"paragraph","content":[{"text":"v2.2.1","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[185.0]},"content":[{"type":"paragraph","content":[{"type":"date","attrs":{"timestamp":"1708646400000"}}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[224.0]},"content":[{"type":"paragraph","content":[{"type":"status","attrs":{"color":"blue","style":"bold","text":"IMPROVEMENTS","localId":"e5828b41-f8c3-4097-b576-14c6747a3b4d"}}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[222.0]},"content":[{"type":"paragraph","content":[{"text":"Update","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":3,"rowspan":1,"colwidth":[185.0,224.0,222.0]},"content":[{"type":"nestedExpand","attrs":{"title":"Details"},"content":[{"type":"paragraph","content":[{"text":"Improvements","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Upgraded the DCSDK from 1.10.3 to 1.11.0:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Updated DevoSDK to v5.1.10","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Fix for SyslogSender related to UTF-8","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Enhance of troubleshooting. Trace Standardization, Some traces has been introduced.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Introduced a machanism to detect \"Out of Memory killer\" situation.","type":"text"}]}]}]}]}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":2,"colwidth":[129.0]},"content":[{"type":"paragraph","content":[{"text":"v2.2.0","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[185.0]},"content":[{"type":"paragraph","content":[{"type":"date","attrs":{"timestamp":"1705363200000"}}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[224.0]},"content":[{"type":"paragraph","content":[{"type":"status","attrs":{"color":"blue","style":"bold","text":"IMPROVEMENTS","localId":"19eae8bb-079b-471b-834b-c01be4e9ae95"}},{"type":"hardBreak"},{"type":"status","attrs":{"color":"yellow","style":"bold","text":"BUG fixing","localId":"0c48f0b8-4215-4c93-885a-4ac571730019"}}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[222.0]},"content":[{"type":"paragraph","content":[{"text":"Update","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":3,"rowspan":1,"colwidth":[185.0,224.0,222.0]},"content":[{"type":"nestedExpand","attrs":{"title":"Details"},"content":[{"type":"paragraph","content":[{"text":"Improvements","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Upgraded the DCSDK docker base image version to 1.1.0","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Bugs","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Fixed the issue where the collector is unable to fetch data after a certain time period","type":"text"}]}]}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#ffffff","rowspan":2,"colwidth":[129.0]},"content":[{"type":"paragraph","content":[{"text":"v2.1.1","type":"text","marks":[{"type":"code"}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#ffffff","rowspan":1,"colwidth":[185.0]},"content":[{"type":"paragraph","content":[{"type":"date","attrs":{"timestamp":"1703116800000"}}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#ffffff","rowspan":1,"colwidth":[224.0]},"content":[{"type":"paragraph","content":[{"type":"status","attrs":{"color":"purple","style":"bold","text":"FIRST RELEASE","localId":"964fcc68-68ed-46e8-8e43-1f9af0a97373"}},{"text":" ","type":"text"},{"type":"hardBreak"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#ffffff","rowspan":1,"colwidth":[222.0]},"content":[{"type":"paragraph","content":[{"text":"Update","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":" ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":3,"background":"#ffffff","rowspan":1,"colwidth":[185.0,224.0,222.0]},"content":[{"type":"nestedExpand","attrs":{"title":"Details"},"content":[{"type":"paragraph","content":[{"text":"Released the first version of the Proofpoint TAP collector.","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Version migration","type":"text"}]},{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"children","parameters":{"macroParams":{"allChildren":{"value":"true"}},"macroMetadata":{"macroId":{"value":"660d981a-e6fd-4d0c-b423-ebe67c69aee0"},"schemaVersion":{"value":"2"},"title":"Child pages (Children Display)"}},"localId":"93b1c9e7-3db8-47c7-b2d9-e9a57df912c2"}}],"version":1}

Browser not supported