Skip to content

Fix logger in SdJournal, add optional filters and Docker journald tags #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
marcogroppo wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Fix logger in SdJournal, add optional filters and Docker journald tags #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5ee699b
Use the new logger in SdJournal.
marcogroppo Mar 19, 2017
591f0ad
Add optional logging filters and send docker-generated journald fields.
marcogroppo Mar 19, 2017
e7a742f
Fix idle logging.
marcogroppo Mar 22, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cloud-watch/cloudwatch_journal_repeater.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@ func (repeater *CloudWatchJournalRepeater) WriteBatch(records []*Record) error {
}

if len(describeOutput.LogStreams) > 0 {
if describeOutput.LogStreams[0].UploadSequenceToken == nil {
return nil
}
repeater.nextSequenceToken =
*describeOutput.LogStreams[0].UploadSequenceToken

Expand Down
1 change: 1 addition & 0 deletions cloud-watch/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ type Config struct {
Tail bool `hcl:"tail"`
Rewind int `hcl:"rewind"`
Local bool `hcl:"local"`
Filters []string `hcl:"filters"`
AllowedFields []string `hcl:"fields"`
OmitFields []string `hcl:"omit_fields"`
logPriority int
Expand Down
2 changes: 1 addition & 1 deletion cloud-watch/creators.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func CreateRepeater(config *Config, logger lg.Logger) JournalRepeater {
var err error

if !config.MockCloudWatch {
logger.Info("Creating repeater that is conneting to AWS cloud watch")
logger.Info("Creating repeater that is connecting to AWS cloud watch")
session := NewAWSSession(config)
repeater, err = NewCloudWatchJournalRepeater(session, nil, config)

Expand Down
15 changes: 11 additions & 4 deletions cloud-watch/journal_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,18 @@ import (
"github.com/coreos/go-systemd/sdjournal"
"strconv"
"time"
lg "github.com/advantageous/go-logback/logging"
)

type SdJournal struct {
journal *sdjournal.Journal
logger *Logger
logger lg.Logger
debug bool
}

func NewJournal(config *Config) (Journal, error) {

logger := NewSimpleLogger("journal", config)
logger := lg.NewSimpleLogger("journal")

var debug bool

Expand All @@ -30,7 +31,7 @@ func NewJournal(config *Config) (Journal, error) {
journal, logger, debug,
}, err
} else {
logger.Info.Printf("using journal dir: %s", config.JournalDir)
logger.Infof("using journal dir: %s", config.JournalDir)
journal, err := sdjournal.NewJournalFromDir(config.JournalDir)

return &SdJournal{
Expand All @@ -51,6 +52,12 @@ func (journal *SdJournal) AddLogFilters(config *Config) {
}
journal.journal.AddDisjunction()
}
// Add other Filters
if config.Filters != nil && len(config.Filters) > 0 {
for _, filter := range config.Filters {
journal.journal.AddMatch(filter)
}
}
}

func (journal *SdJournal) Close() error {
Expand All @@ -61,7 +68,7 @@ func (journal *SdJournal) Close() error {
func (journal *SdJournal) Next() (uint64, error) {
loc, err := journal.journal.Next()
if journal.debug {
journal.logger.Info.Printf("NEXT location %d %v", loc, err)
journal.logger.Infof("NEXT location %d %v", loc, err)
}

return loc, err
Expand Down
54 changes: 29 additions & 25 deletions cloud-watch/record.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,31 +32,35 @@ var PriorityJsonMap = map[Priority][]byte{
}

type Record struct {
InstanceId string `json:"instanceId,omitempty"`
TimeUsec int64 `json:"-" journald:"__REALTIME_TIMESTAMP"`
PID int `json:"pid,omitempty" journald:"_PID"`
UID int `json:"uid,omitempty" journald:"_UID"`
GID int `json:"gid,omitempty" journald:"_GID"`
Command string `json:"cmdName,omitempty" journald:"_COMM"`
Executable string `json:"exe,omitempty" journald:"_EXE"`
CommandLine string `json:"cmdLine,omitempty" journald:"_CMDLINE"`
SystemdUnit string `json:"systemdUnit,omitempty" journald:"_SYSTEMD_UNIT"`
BootId string `json:"bootId,omitempty" journald:"_BOOT_ID"`
MachineId string `json:"machineId,omitempty" journald:"_MACHINE_ID"`
Hostname string `json:"hostname,omitempty" journald:"_HOSTNAME"`
Transport string `json:"transport,omitempty" journald:"_TRANSPORT"`
Priority Priority `json:"priority" journald:"PRIORITY"`
Message string `json:"message" journald:"MESSAGE"`
MessageId string `json:"messageId,omitempty" journald:"MESSAGE_ID"`
Errno int `json:"machineId,omitempty" journald:"ERRNO"`
SeqId int64 `json:"seq,omitempty" `
Facility int `json:"syslogFacility,omitempty" journald:"SYSLOG_FACILITY"`
Identifier string `json:"syslogIdent,omitempty" journald:"SYSLOG_IDENTIFIER"`
SysPID int `json:"syslogPid,omitempty" journald:"SYSLOG_PID"`
Device string `json:"kernelDevice,omitempty" journald:"_KERNEL_DEVICE"`
Subsystem string `json:"kernelSubsystem,omitempty" journald:"_KERNEL_SUBSYSTEM"`
SysName string `json:"kernelSysName,omitempty" journald:"_UDEV_SYSNAME"`
DevNode string `json:"kernelDevNode,omitempty" journald:"_UDEV_DEVNODE"`
InstanceId string `json:"instanceId,omitempty"`
TimeUsec int64 `json:"-" journald:"__REALTIME_TIMESTAMP"`
PID int `json:"pid,omitempty" journald:"_PID"`
UID int `json:"uid,omitempty" journald:"_UID"`
GID int `json:"gid,omitempty" journald:"_GID"`
Command string `json:"cmdName,omitempty" journald:"_COMM"`
Executable string `json:"exe,omitempty" journald:"_EXE"`
CommandLine string `json:"cmdLine,omitempty" journald:"_CMDLINE"`
SystemdUnit string `json:"systemdUnit,omitempty" journald:"_SYSTEMD_UNIT"`
BootId string `json:"bootId,omitempty" journald:"_BOOT_ID"`
MachineId string `json:"machineId,omitempty" journald:"_MACHINE_ID"`
Hostname string `json:"hostname,omitempty" journald:"_HOSTNAME"`
Transport string `json:"transport,omitempty" journald:"_TRANSPORT"`
Priority Priority `json:"priority" journald:"PRIORITY"`
Message string `json:"message" journald:"MESSAGE"`
MessageId string `json:"messageId,omitempty" journald:"MESSAGE_ID"`
Errno int `json:"machineId,omitempty" journald:"ERRNO"`
SeqId int64 `json:"seq,omitempty" `
Facility int `json:"syslogFacility,omitempty" journald:"SYSLOG_FACILITY"`
Identifier string `json:"syslogIdent,omitempty" journald:"SYSLOG_IDENTIFIER"`
SysPID int `json:"syslogPid,omitempty" journald:"SYSLOG_PID"`
Device string `json:"kernelDevice,omitempty" journald:"_KERNEL_DEVICE"`
Subsystem string `json:"kernelSubsystem,omitempty" journald:"_KERNEL_SUBSYSTEM"`
SysName string `json:"kernelSysName,omitempty" journald:"_UDEV_SYSNAME"`
DevNode string `json:"kernelDevNode,omitempty" journald:"_UDEV_DEVNODE"`
ContainerId string `json:"containerId,omitempty" journald:"CONTAINER_ID"`
ContainerIdFull string `json:"containerIdFull,omitempty" journald:"CONTAINER_ID_FULL"`
ContainerName string `json:"containerName,omitempty" journald:"CONTAINER_NAME"`
ContainerTag string `json:"containerTag,omitempty" journald:"CONTAINER_TAG"`
}

func NewRecord(journal Journal, logger lg.Logger, config *Config) (*Record, error) {
Expand Down
2 changes: 1 addition & 1 deletion cloud-watch/workers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func NewRunnerInternal(journal Journal, repeater JournalRepeater, logger lg.Logg
r.sendBatch()
now := time.Now().Unix()
if now-r.lastMetricTime > 120 {
now = r.lastMetricTime
r.lastMetricTime = now
r.logger.Infof("Systemd CloudWatch: batches sent %d, idleCount %d, emptyCount %d",
r.batchCounter, r.idleCounter, r.emptyCounter)
}
Expand Down